1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
9 #define NIX_TX_OFFLOAD_NONE (0)
10 #define NIX_TX_OFFLOAD_L3_L4_CSUM_F BIT(0)
11 #define NIX_TX_OFFLOAD_OL3_OL4_CSUM_F BIT(1)
12 #define NIX_TX_OFFLOAD_VLAN_QINQ_F BIT(2)
13 #define NIX_TX_OFFLOAD_MBUF_NOFF_F BIT(3)
14 #define NIX_TX_OFFLOAD_TSO_F BIT(4)
15 #define NIX_TX_OFFLOAD_TSTAMP_F BIT(5)
16 #define NIX_TX_OFFLOAD_SECURITY_F BIT(6)
18 /* Flags to control xmit_prepare function.
19 * Defining it from backwards to denote its been
20 * not used as offload flags to pick function
22 #define NIX_TX_MULTI_SEG_F BIT(15)
24 #define NIX_TX_NEED_SEND_HDR_W1 \
25 (NIX_TX_OFFLOAD_L3_L4_CSUM_F | NIX_TX_OFFLOAD_OL3_OL4_CSUM_F | \
26 NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F)
28 #define NIX_TX_NEED_EXT_HDR \
29 (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSTAMP_F | \
32 #define NIX_XMIT_FC_OR_RETURN(txq, pkts) \
34 /* Cached value is low, Update the fc_cache_pkts */ \
35 if (unlikely((txq)->fc_cache_pkts < (pkts))) { \
36 /* Multiply with sqe_per_sqb to express in pkts */ \
37 (txq)->fc_cache_pkts = \
38 ((txq)->nb_sqb_bufs_adj - *(txq)->fc_mem) \
39 << (txq)->sqes_per_sqb_log2; \
40 /* Check it again for the room */ \
41 if (unlikely((txq)->fc_cache_pkts < (pkts))) \
46 /* Function to determine no of tx subdesc required in case ext
47 * sub desc is enabled.
49 static __rte_always_inline int
50 cn9k_nix_tx_ext_subs(const uint16_t flags)
52 return (flags & NIX_TX_OFFLOAD_TSTAMP_F)
55 (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F))
60 static __rte_always_inline void
61 cn9k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags)
63 uint64_t mask, ol_flags = m->ol_flags;
65 if (flags & NIX_TX_OFFLOAD_TSO_F && (ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
66 uintptr_t mdata = rte_pktmbuf_mtod(m, uintptr_t);
67 uint16_t *iplen, *oiplen, *oudplen;
68 uint16_t lso_sb, paylen;
70 mask = -!!(ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IPV6));
71 lso_sb = (mask & (m->outer_l2_len + m->outer_l3_len)) +
72 m->l2_len + m->l3_len + m->l4_len;
74 /* Reduce payload len from base headers */
75 paylen = m->pkt_len - lso_sb;
77 /* Get iplen position assuming no tunnel hdr */
78 iplen = (uint16_t *)(mdata + m->l2_len +
79 (2 << !!(ol_flags & RTE_MBUF_F_TX_IPV6)));
80 /* Handle tunnel tso */
81 if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
82 (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) {
83 const uint8_t is_udp_tun =
84 (CNXK_NIX_UDP_TUN_BITMASK >>
85 ((ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) >> 45)) &
88 oiplen = (uint16_t *)(mdata + m->outer_l2_len +
90 RTE_MBUF_F_TX_OUTER_IPV6)));
91 *oiplen = rte_cpu_to_be_16(rte_be_to_cpu_16(*oiplen) -
94 /* Update format for UDP tunneled packet */
96 oudplen = (uint16_t *)(mdata + m->outer_l2_len +
98 *oudplen = rte_cpu_to_be_16(
99 rte_be_to_cpu_16(*oudplen) - paylen);
102 /* Update iplen position to inner ip hdr */
103 iplen = (uint16_t *)(mdata + lso_sb - m->l3_len -
105 (2 << !!(ol_flags & RTE_MBUF_F_TX_IPV6)));
108 *iplen = rte_cpu_to_be_16(rte_be_to_cpu_16(*iplen) - paylen);
112 static __rte_always_inline void
113 cn9k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags,
114 const uint64_t lso_tun_fmt)
116 struct nix_send_ext_s *send_hdr_ext;
117 struct nix_send_hdr_s *send_hdr;
118 uint64_t ol_flags = 0, mask;
119 union nix_send_hdr_w1_u w1;
120 union nix_send_sg_s *sg;
122 send_hdr = (struct nix_send_hdr_s *)cmd;
123 if (flags & NIX_TX_NEED_EXT_HDR) {
124 send_hdr_ext = (struct nix_send_ext_s *)(cmd + 2);
125 sg = (union nix_send_sg_s *)(cmd + 4);
126 /* Clear previous markings */
127 send_hdr_ext->w0.lso = 0;
128 send_hdr_ext->w1.u = 0;
130 sg = (union nix_send_sg_s *)(cmd + 2);
133 if (flags & NIX_TX_NEED_SEND_HDR_W1) {
134 ol_flags = m->ol_flags;
138 if (!(flags & NIX_TX_MULTI_SEG_F)) {
139 send_hdr->w0.total = m->data_len;
141 roc_npa_aura_handle_to_aura(m->pool->pool_id);
146 * 3 => IPV4 with csum
148 * L3type and L3ptr needs to be set for either
149 * L3 csum or L4 csum or LSO
153 if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
154 (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) {
155 const uint8_t csum = !!(ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM);
156 const uint8_t ol3type =
157 ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4)) << 1) +
158 ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)) << 2) +
159 !!(ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM);
162 w1.ol3type = ol3type;
163 mask = 0xffffull << ((!!ol3type) << 4);
164 w1.ol3ptr = ~mask & m->outer_l2_len;
165 w1.ol4ptr = ~mask & (w1.ol3ptr + m->outer_l3_len);
168 w1.ol4type = csum + (csum << 1);
171 w1.il3type = ((!!(ol_flags & RTE_MBUF_F_TX_IPV4)) << 1) +
172 ((!!(ol_flags & RTE_MBUF_F_TX_IPV6)) << 2);
173 w1.il3ptr = w1.ol4ptr + m->l2_len;
174 w1.il4ptr = w1.il3ptr + m->l3_len;
175 /* Increment it by 1 if it is IPV4 as 3 is with csum */
176 w1.il3type = w1.il3type + !!(ol_flags & RTE_MBUF_F_TX_IP_CKSUM);
179 w1.il4type = (ol_flags & RTE_MBUF_F_TX_L4_MASK) >> 52;
181 /* In case of no tunnel header use only
182 * shift IL3/IL4 fields a bit to use
183 * OL3/OL4 for header checksum
186 w1.u = ((w1.u & 0xFFFFFFFF00000000) >> (mask << 3)) |
187 ((w1.u & 0X00000000FFFFFFFF) >> (mask << 4));
189 } else if (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) {
190 const uint8_t csum = !!(ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM);
191 const uint8_t outer_l2_len = m->outer_l2_len;
194 w1.ol3ptr = outer_l2_len;
195 w1.ol4ptr = outer_l2_len + m->outer_l3_len;
196 /* Increment it by 1 if it is IPV4 as 3 is with csum */
197 w1.ol3type = ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4)) << 1) +
198 ((!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)) << 2) +
199 !!(ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM);
202 w1.ol4type = csum + (csum << 1);
204 } else if (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) {
205 const uint8_t l2_len = m->l2_len;
207 /* Always use OLXPTR and OLXTYPE when only
208 * when one header is present
213 w1.ol4ptr = l2_len + m->l3_len;
214 /* Increment it by 1 if it is IPV4 as 3 is with csum */
215 w1.ol3type = ((!!(ol_flags & RTE_MBUF_F_TX_IPV4)) << 1) +
216 ((!!(ol_flags & RTE_MBUF_F_TX_IPV6)) << 2) +
217 !!(ol_flags & RTE_MBUF_F_TX_IP_CKSUM);
220 w1.ol4type = (ol_flags & RTE_MBUF_F_TX_L4_MASK) >> 52;
223 if (flags & NIX_TX_NEED_EXT_HDR && flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
224 send_hdr_ext->w1.vlan1_ins_ena = !!(ol_flags & RTE_MBUF_F_TX_VLAN);
225 /* HW will update ptr after vlan0 update */
226 send_hdr_ext->w1.vlan1_ins_ptr = 12;
227 send_hdr_ext->w1.vlan1_ins_tci = m->vlan_tci;
229 send_hdr_ext->w1.vlan0_ins_ena = !!(ol_flags & RTE_MBUF_F_TX_QINQ);
230 /* 2B before end of l2 header */
231 send_hdr_ext->w1.vlan0_ins_ptr = 12;
232 send_hdr_ext->w1.vlan0_ins_tci = m->vlan_tci_outer;
235 if (flags & NIX_TX_OFFLOAD_TSO_F && (ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
239 mask = -(!w1.il3type);
240 lso_sb = (mask & w1.ol4ptr) + (~mask & w1.il4ptr) + m->l4_len;
242 send_hdr_ext->w0.lso_sb = lso_sb;
243 send_hdr_ext->w0.lso = 1;
244 send_hdr_ext->w0.lso_mps = m->tso_segsz;
245 send_hdr_ext->w0.lso_format =
246 NIX_LSO_FORMAT_IDX_TSOV4 + !!(ol_flags & RTE_MBUF_F_TX_IPV6);
247 w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
249 /* Handle tunnel tso */
250 if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
251 (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) {
252 const uint8_t is_udp_tun =
253 (CNXK_NIX_UDP_TUN_BITMASK >>
254 ((ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) >> 45)) &
256 uint8_t shift = is_udp_tun ? 32 : 0;
258 shift += (!!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) << 4);
259 shift += (!!(ol_flags & RTE_MBUF_F_TX_IPV6) << 3);
261 w1.il4type = NIX_SENDL4TYPE_TCP_CKSUM;
262 w1.ol4type = is_udp_tun ? NIX_SENDL4TYPE_UDP_CKSUM : 0;
263 /* Update format for UDP tunneled packet */
264 send_hdr_ext->w0.lso_format = (lso_tun_fmt >> shift);
268 if (flags & NIX_TX_NEED_SEND_HDR_W1)
269 send_hdr->w1.u = w1.u;
271 if (!(flags & NIX_TX_MULTI_SEG_F)) {
272 sg->seg1_size = m->data_len;
273 *(rte_iova_t *)(++sg) = rte_mbuf_data_iova(m);
275 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
276 /* DF bit = 1 if refcount of current mbuf or parent mbuf
278 * DF bit = 0 otherwise
280 send_hdr->w0.df = cnxk_nix_prefree_seg(m);
281 /* Ensuring mbuf fields which got updated in
282 * cnxk_nix_prefree_seg are written before LMTST.
286 /* Mark mempool object as "put" since it is freed by NIX */
287 if (!send_hdr->w0.df)
288 RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
292 static __rte_always_inline void
293 cn9k_nix_xmit_prepare_tstamp(uint64_t *cmd, const uint64_t *send_mem_desc,
294 const uint64_t ol_flags, const uint16_t no_segdw,
295 const uint16_t flags)
297 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
298 struct nix_send_mem_s *send_mem;
299 uint16_t off = (no_segdw - 1) << 1;
300 const uint8_t is_ol_tstamp = !(ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST);
302 send_mem = (struct nix_send_mem_s *)(cmd + off);
303 if (flags & NIX_TX_MULTI_SEG_F) {
304 /* Retrieving the default desc values */
305 cmd[off] = send_mem_desc[6];
307 /* Using compiler barier to avoid voilation of C
310 rte_compiler_barrier();
313 /* Packets for which RTE_MBUF_F_TX_IEEE1588_TMST is not set, tx tstamp
314 * should not be recorded, hence changing the alg type to
315 * NIX_SENDMEMALG_SET and also changing send mem addr field to
316 * next 8 bytes as it corrpt the actual tx tstamp registered
319 send_mem->w0.cn9k.alg =
320 NIX_SENDMEMALG_SETTSTMP - (is_ol_tstamp);
322 send_mem->addr = (rte_iova_t)((uint64_t *)send_mem_desc[7] +
327 static __rte_always_inline void
328 cn9k_nix_xmit_one(uint64_t *cmd, void *lmt_addr, const rte_iova_t io_addr,
329 const uint32_t flags)
334 roc_lmt_mov(lmt_addr, cmd, cn9k_nix_tx_ext_subs(flags));
335 lmt_status = roc_lmt_submit_ldeor(io_addr);
336 } while (lmt_status == 0);
339 static __rte_always_inline void
340 cn9k_nix_xmit_prep_lmt(uint64_t *cmd, void *lmt_addr, const uint32_t flags)
342 roc_lmt_mov(lmt_addr, cmd, cn9k_nix_tx_ext_subs(flags));
345 static __rte_always_inline uint64_t
346 cn9k_nix_xmit_submit_lmt(const rte_iova_t io_addr)
348 return roc_lmt_submit_ldeor(io_addr);
351 static __rte_always_inline uint64_t
352 cn9k_nix_xmit_submit_lmt_release(const rte_iova_t io_addr)
354 return roc_lmt_submit_ldeorl(io_addr);
357 static __rte_always_inline uint16_t
358 cn9k_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
360 struct nix_send_hdr_s *send_hdr;
361 union nix_send_sg_s *sg;
362 struct rte_mbuf *m_next;
363 uint64_t *slist, sg_u;
368 send_hdr = (struct nix_send_hdr_s *)cmd;
369 send_hdr->w0.total = m->pkt_len;
370 send_hdr->w0.aura = roc_npa_aura_handle_to_aura(m->pool->pool_id);
372 if (flags & NIX_TX_NEED_EXT_HDR)
377 sg = (union nix_send_sg_s *)&cmd[2 + off];
378 /* Clear sg->u header before use */
379 sg->u &= 0xFC00000000000000;
381 slist = &cmd[3 + off];
384 nb_segs = m->nb_segs;
386 /* Fill mbuf segments */
389 sg_u = sg_u | ((uint64_t)m->data_len << (i << 4));
390 *slist = rte_mbuf_data_iova(m);
391 /* Set invert df if buffer is not to be freed by H/W */
392 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
393 sg_u |= (cnxk_nix_prefree_seg(m) << (i + 55));
394 /* Commit changes to mbuf */
397 /* Mark mempool object as "put" since it is freed by NIX */
398 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
399 if (!(sg_u & (1ULL << (i + 55))))
400 RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
406 if (i > 2 && nb_segs) {
408 /* Next SG subdesc */
409 *(uint64_t *)slist = sg_u & 0xFC00000000000000;
412 sg = (union nix_send_sg_s *)slist;
421 segdw = (uint64_t *)slist - (uint64_t *)&cmd[2 + off];
422 /* Roundup extra dwords to multiple of 2 */
423 segdw = (segdw >> 1) + (segdw & 0x1);
425 segdw += (off >> 1) + 1 + !!(flags & NIX_TX_OFFLOAD_TSTAMP_F);
426 send_hdr->w0.sizem1 = segdw - 1;
431 static __rte_always_inline void
432 cn9k_nix_xmit_mseg_prep_lmt(uint64_t *cmd, void *lmt_addr, uint16_t segdw)
434 roc_lmt_mov_seg(lmt_addr, (const void *)cmd, segdw);
437 static __rte_always_inline void
438 cn9k_nix_xmit_mseg_one(uint64_t *cmd, void *lmt_addr, rte_iova_t io_addr,
444 roc_lmt_mov_seg(lmt_addr, (const void *)cmd, segdw);
445 lmt_status = roc_lmt_submit_ldeor(io_addr);
446 } while (lmt_status == 0);
449 static __rte_always_inline void
450 cn9k_nix_xmit_mseg_one_release(uint64_t *cmd, void *lmt_addr,
451 rte_iova_t io_addr, uint16_t segdw)
457 roc_lmt_mov_seg(lmt_addr, (const void *)cmd, segdw);
458 lmt_status = roc_lmt_submit_ldeor(io_addr);
459 } while (lmt_status == 0);
462 static __rte_always_inline uint16_t
463 cn9k_nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts,
464 uint64_t *cmd, const uint16_t flags)
466 struct cn9k_eth_txq *txq = tx_queue;
467 const rte_iova_t io_addr = txq->io_addr;
468 void *lmt_addr = txq->lmt_addr;
469 uint64_t lso_tun_fmt;
472 NIX_XMIT_FC_OR_RETURN(txq, pkts);
474 roc_lmt_mov(cmd, &txq->cmd[0], cn9k_nix_tx_ext_subs(flags));
476 /* Perform header writes before barrier for TSO */
477 if (flags & NIX_TX_OFFLOAD_TSO_F) {
478 lso_tun_fmt = txq->lso_tun_fmt;
480 for (i = 0; i < pkts; i++)
481 cn9k_nix_xmit_prepare_tso(tx_pkts[i], flags);
484 /* Lets commit any changes in the packet here as no further changes
485 * to the packet will be done unless no fast free is enabled.
487 if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F))
490 for (i = 0; i < pkts; i++) {
491 cn9k_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt);
492 cn9k_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
493 tx_pkts[i]->ol_flags, 4, flags);
494 cn9k_nix_xmit_one(cmd, lmt_addr, io_addr, flags);
497 /* Reduce the cached count */
498 txq->fc_cache_pkts -= pkts;
503 static __rte_always_inline uint16_t
504 cn9k_nix_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts,
505 uint16_t pkts, uint64_t *cmd, const uint16_t flags)
507 struct cn9k_eth_txq *txq = tx_queue;
508 const rte_iova_t io_addr = txq->io_addr;
509 void *lmt_addr = txq->lmt_addr;
510 uint64_t lso_tun_fmt;
514 NIX_XMIT_FC_OR_RETURN(txq, pkts);
516 roc_lmt_mov(cmd, &txq->cmd[0], cn9k_nix_tx_ext_subs(flags));
518 /* Perform header writes before barrier for TSO */
519 if (flags & NIX_TX_OFFLOAD_TSO_F) {
520 lso_tun_fmt = txq->lso_tun_fmt;
522 for (i = 0; i < pkts; i++)
523 cn9k_nix_xmit_prepare_tso(tx_pkts[i], flags);
526 /* Lets commit any changes in the packet here as no further changes
527 * to the packet will be done unless no fast free is enabled.
529 if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F))
532 for (i = 0; i < pkts; i++) {
533 cn9k_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt);
534 segdw = cn9k_nix_prepare_mseg(tx_pkts[i], cmd, flags);
535 cn9k_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
536 tx_pkts[i]->ol_flags, segdw,
538 cn9k_nix_xmit_mseg_one(cmd, lmt_addr, io_addr, segdw);
541 /* Reduce the cached count */
542 txq->fc_cache_pkts -= pkts;
547 #if defined(RTE_ARCH_ARM64)
549 static __rte_always_inline void
550 cn9k_nix_prepare_tso(struct rte_mbuf *m, union nix_send_hdr_w1_u *w1,
551 union nix_send_ext_w0_u *w0, uint64_t ol_flags,
557 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG))
560 mask = -(!w1->il3type);
561 lso_sb = (mask & w1->ol4ptr) + (~mask & w1->il4ptr) + m->l4_len;
565 w0->lso_mps = m->tso_segsz;
566 w0->lso_format = NIX_LSO_FORMAT_IDX_TSOV4 + !!(ol_flags & RTE_MBUF_F_TX_IPV6);
567 w1->ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
569 /* Handle tunnel tso */
570 if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
571 (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) {
572 const uint8_t is_udp_tun =
573 (CNXK_NIX_UDP_TUN_BITMASK >>
574 ((ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) >> 45)) &
577 w1->il4type = NIX_SENDL4TYPE_TCP_CKSUM;
578 w1->ol4type = is_udp_tun ? NIX_SENDL4TYPE_UDP_CKSUM : 0;
579 /* Update format for UDP tunneled packet */
580 w0->lso_format += is_udp_tun ? 2 : 6;
582 w0->lso_format += !!(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) << 1;
586 static __rte_always_inline uint8_t
587 cn9k_nix_prepare_mseg_vec_list(struct rte_mbuf *m, uint64_t *cmd,
588 union nix_send_hdr_w0_u *sh,
589 union nix_send_sg_s *sg, const uint32_t flags)
591 struct rte_mbuf *m_next;
592 uint64_t *slist, sg_u;
597 sh->total = m->pkt_len;
598 /* Clear sg->u header before use */
599 sg->u &= 0xFC00000000000000;
603 sg_u = sg_u | ((uint64_t)m->data_len);
605 nb_segs = m->nb_segs - 1;
608 /* Set invert df if buffer is not to be freed by H/W */
609 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)
610 sg_u |= (cnxk_nix_prefree_seg(m) << 55);
611 /* Mark mempool object as "put" since it is freed by NIX */
612 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
613 if (!(sg_u & (1ULL << 55)))
614 RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
619 /* Fill mbuf segments */
622 sg_u = sg_u | ((uint64_t)m->data_len << (i << 4));
623 *slist = rte_mbuf_data_iova(m);
624 /* Set invert df if buffer is not to be freed by H/W */
625 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)
626 sg_u |= (cnxk_nix_prefree_seg(m) << (i + 55));
627 /* Mark mempool object as "put" since it is freed by NIX
629 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
630 if (!(sg_u & (1ULL << (i + 55))))
631 RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
637 if (i > 2 && nb_segs) {
639 /* Next SG subdesc */
640 *(uint64_t *)slist = sg_u & 0xFC00000000000000;
643 sg = (union nix_send_sg_s *)slist;
652 segdw = (uint64_t *)slist - (uint64_t *)&cmd[0];
655 /* Roundup extra dwords to multiple of 2 */
656 segdw = (segdw >> 1) + (segdw & 0x1);
658 segdw += 1 + !!(flags & NIX_TX_NEED_EXT_HDR) +
659 !!(flags & NIX_TX_OFFLOAD_TSTAMP_F);
660 sh->sizem1 = segdw - 1;
665 static __rte_always_inline uint8_t
666 cn9k_nix_prepare_mseg_vec(struct rte_mbuf *m, uint64_t *cmd, uint64x2_t *cmd0,
667 uint64x2_t *cmd1, const uint32_t flags)
669 union nix_send_hdr_w0_u sh;
670 union nix_send_sg_s sg;
673 if (m->nb_segs == 1) {
674 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
675 sg.u = vgetq_lane_u64(cmd1[0], 0);
676 sg.u |= (cnxk_nix_prefree_seg(m) << 55);
677 cmd1[0] = vsetq_lane_u64(sg.u, cmd1[0], 0);
680 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
681 sg.u = vgetq_lane_u64(cmd1[0], 0);
682 if (!(sg.u & (1ULL << 55)))
683 RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
686 return 2 + !!(flags & NIX_TX_NEED_EXT_HDR) +
687 !!(flags & NIX_TX_OFFLOAD_TSTAMP_F);
690 sh.u = vgetq_lane_u64(cmd0[0], 0);
691 sg.u = vgetq_lane_u64(cmd1[0], 0);
693 ret = cn9k_nix_prepare_mseg_vec_list(m, cmd, &sh, &sg, flags);
695 cmd0[0] = vsetq_lane_u64(sh.u, cmd0[0], 0);
696 cmd1[0] = vsetq_lane_u64(sg.u, cmd1[0], 0);
700 #define NIX_DESCS_PER_LOOP 4
702 static __rte_always_inline void
703 cn9k_nix_xmit_pkts_mseg_vector(uint64x2_t *cmd0, uint64x2_t *cmd1,
704 uint64x2_t *cmd2, uint64x2_t *cmd3,
706 uint64_t slist[][CNXK_NIX_TX_MSEG_SG_DWORDS - 2],
707 uint64_t *lmt_addr, rte_iova_t io_addr,
708 const uint32_t flags)
713 if (!(flags & NIX_TX_NEED_EXT_HDR) &&
714 !(flags & NIX_TX_OFFLOAD_TSTAMP_F)) {
715 /* No segments in 4 consecutive packets. */
716 if ((segdw[0] + segdw[1] + segdw[2] + segdw[3]) <= 8) {
718 vst1q_u64(lmt_addr, cmd0[0]);
719 vst1q_u64(lmt_addr + 2, cmd1[0]);
720 vst1q_u64(lmt_addr + 4, cmd0[1]);
721 vst1q_u64(lmt_addr + 6, cmd1[1]);
722 vst1q_u64(lmt_addr + 8, cmd0[2]);
723 vst1q_u64(lmt_addr + 10, cmd1[2]);
724 vst1q_u64(lmt_addr + 12, cmd0[3]);
725 vst1q_u64(lmt_addr + 14, cmd1[3]);
726 lmt_status = roc_lmt_submit_ldeor(io_addr);
727 } while (lmt_status == 0);
733 for (j = 0; j < NIX_DESCS_PER_LOOP;) {
734 /* Fit consecutive packets in same LMTLINE. */
735 if ((segdw[j] + segdw[j + 1]) <= 8) {
737 if ((flags & NIX_TX_NEED_EXT_HDR) &&
738 (flags & NIX_TX_OFFLOAD_TSTAMP_F)) {
739 vst1q_u64(lmt_addr, cmd0[j]);
740 vst1q_u64(lmt_addr + 2, cmd2[j]);
741 vst1q_u64(lmt_addr + 4, cmd1[j]);
744 roc_lmt_mov_seg(lmt_addr + 6, slist[j], off);
746 vst1q_u64(lmt_addr + 6 + off, cmd3[j]);
748 vst1q_u64(lmt_addr + 8 + off, cmd0[j + 1]);
749 vst1q_u64(lmt_addr + 10 + off, cmd2[j + 1]);
750 vst1q_u64(lmt_addr + 12 + off, cmd1[j + 1]);
751 roc_lmt_mov_seg(lmt_addr + 14 + off,
752 slist[j + 1], segdw[j + 1] - 4);
753 off += ((segdw[j + 1] - 4) << 1);
754 vst1q_u64(lmt_addr + 14 + off, cmd3[j + 1]);
755 } else if (flags & NIX_TX_NEED_EXT_HDR) {
756 vst1q_u64(lmt_addr, cmd0[j]);
757 vst1q_u64(lmt_addr + 2, cmd2[j]);
758 vst1q_u64(lmt_addr + 4, cmd1[j]);
761 roc_lmt_mov_seg(lmt_addr + 6, slist[j], off);
763 vst1q_u64(lmt_addr + 6 + off, cmd0[j + 1]);
764 vst1q_u64(lmt_addr + 8 + off, cmd2[j + 1]);
765 vst1q_u64(lmt_addr + 10 + off, cmd1[j + 1]);
766 roc_lmt_mov_seg(lmt_addr + 12 + off,
767 slist[j + 1], segdw[j + 1] - 3);
769 vst1q_u64(lmt_addr, cmd0[j]);
770 vst1q_u64(lmt_addr + 2, cmd1[j]);
773 roc_lmt_mov_seg(lmt_addr + 4, slist[j], off);
775 vst1q_u64(lmt_addr + 4 + off, cmd0[j + 1]);
776 vst1q_u64(lmt_addr + 6 + off, cmd1[j + 1]);
777 roc_lmt_mov_seg(lmt_addr + 8 + off,
778 slist[j + 1], segdw[j + 1] - 2);
780 lmt_status = roc_lmt_submit_ldeor(io_addr);
786 if ((flags & NIX_TX_NEED_EXT_HDR) &&
787 (flags & NIX_TX_OFFLOAD_TSTAMP_F)) {
788 vst1q_u64(lmt_addr, cmd0[j]);
789 vst1q_u64(lmt_addr + 2, cmd2[j]);
790 vst1q_u64(lmt_addr + 4, cmd1[j]);
793 roc_lmt_mov_seg(lmt_addr + 6, slist[j], off);
795 vst1q_u64(lmt_addr + 6 + off, cmd3[j]);
796 } else if (flags & NIX_TX_NEED_EXT_HDR) {
797 vst1q_u64(lmt_addr, cmd0[j]);
798 vst1q_u64(lmt_addr + 2, cmd2[j]);
799 vst1q_u64(lmt_addr + 4, cmd1[j]);
802 roc_lmt_mov_seg(lmt_addr + 6, slist[j], off);
804 vst1q_u64(lmt_addr, cmd0[j]);
805 vst1q_u64(lmt_addr + 2, cmd1[j]);
808 roc_lmt_mov_seg(lmt_addr + 4, slist[j], off);
810 lmt_status = roc_lmt_submit_ldeor(io_addr);
818 static __rte_always_inline uint16_t
819 cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
820 uint16_t pkts, uint64_t *cmd, const uint16_t flags)
822 uint64x2_t dataoff_iova0, dataoff_iova1, dataoff_iova2, dataoff_iova3;
823 uint64x2_t len_olflags0, len_olflags1, len_olflags2, len_olflags3;
824 uint64x2_t cmd0[NIX_DESCS_PER_LOOP], cmd1[NIX_DESCS_PER_LOOP],
825 cmd2[NIX_DESCS_PER_LOOP], cmd3[NIX_DESCS_PER_LOOP];
826 uint64_t *mbuf0, *mbuf1, *mbuf2, *mbuf3;
827 uint64x2_t senddesc01_w0, senddesc23_w0;
828 uint64x2_t senddesc01_w1, senddesc23_w1;
829 uint64x2_t sendext01_w0, sendext23_w0;
830 uint64x2_t sendext01_w1, sendext23_w1;
831 uint64x2_t sendmem01_w0, sendmem23_w0;
832 uint64x2_t sendmem01_w1, sendmem23_w1;
833 uint64x2_t sgdesc01_w0, sgdesc23_w0;
834 uint64x2_t sgdesc01_w1, sgdesc23_w1;
835 struct cn9k_eth_txq *txq = tx_queue;
836 uint64_t *lmt_addr = txq->lmt_addr;
837 rte_iova_t io_addr = txq->io_addr;
838 uint64x2_t ltypes01, ltypes23;
839 uint64x2_t xtmp128, ytmp128;
840 uint64x2_t xmask01, xmask23;
841 uint64_t lmt_status, i;
844 NIX_XMIT_FC_OR_RETURN(txq, pkts);
846 pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1);
847 pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
849 /* Reduce the cached count */
850 txq->fc_cache_pkts -= pkts;
852 /* Perform header writes before barrier for TSO */
853 if (flags & NIX_TX_OFFLOAD_TSO_F) {
854 for (i = 0; i < pkts; i++)
855 cn9k_nix_xmit_prepare_tso(tx_pkts[i], flags);
858 /* Lets commit any changes in the packet here as no further changes
859 * to the packet will be done unless no fast free is enabled.
861 if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F))
864 senddesc01_w0 = vld1q_dup_u64(&txq->cmd[0]);
865 senddesc23_w0 = senddesc01_w0;
866 senddesc01_w1 = vdupq_n_u64(0);
867 senddesc23_w1 = senddesc01_w1;
869 /* Load command defaults into vector variables. */
870 if (flags & NIX_TX_NEED_EXT_HDR) {
871 sendext01_w0 = vld1q_dup_u64(&txq->cmd[2]);
872 sendext23_w0 = sendext01_w0;
873 sendext01_w1 = vdupq_n_u64(12 | 12U << 24);
874 sendext23_w1 = sendext01_w1;
875 sgdesc01_w0 = vld1q_dup_u64(&txq->cmd[4]);
876 sgdesc23_w0 = sgdesc01_w0;
877 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
878 sendmem01_w0 = vld1q_dup_u64(&txq->cmd[6]);
879 sendmem23_w0 = sendmem01_w0;
880 sendmem01_w1 = vld1q_dup_u64(&txq->cmd[7]);
881 sendmem23_w1 = sendmem01_w1;
884 sgdesc01_w0 = vld1q_dup_u64(&txq->cmd[2]);
885 sgdesc23_w0 = sgdesc01_w0;
888 for (i = 0; i < pkts; i += NIX_DESCS_PER_LOOP) {
889 /* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */
891 vbicq_u64(senddesc01_w0, vdupq_n_u64(0xFFFFFFFF));
892 sgdesc01_w0 = vbicq_u64(sgdesc01_w0, vdupq_n_u64(0xFFFFFFFF));
894 senddesc23_w0 = senddesc01_w0;
895 sgdesc23_w0 = sgdesc01_w0;
897 /* Clear vlan enables. */
898 if (flags & NIX_TX_NEED_EXT_HDR) {
899 sendext01_w1 = vbicq_u64(sendext01_w1,
900 vdupq_n_u64(0x3FFFF00FFFF00));
901 sendext23_w1 = sendext01_w1;
904 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
905 /* Reset send mem alg to SETTSTMP from SUB*/
906 sendmem01_w0 = vbicq_u64(sendmem01_w0,
907 vdupq_n_u64(BIT_ULL(59)));
908 /* Reset send mem address to default. */
910 vbicq_u64(sendmem01_w1, vdupq_n_u64(0xF));
911 sendmem23_w0 = sendmem01_w0;
912 sendmem23_w1 = sendmem01_w1;
915 if (flags & NIX_TX_OFFLOAD_TSO_F) {
916 /* Clear the LSO enable bit. */
917 sendext01_w0 = vbicq_u64(sendext01_w0,
918 vdupq_n_u64(BIT_ULL(14)));
919 sendext23_w0 = sendext01_w0;
922 /* Move mbufs to iova */
923 mbuf0 = (uint64_t *)tx_pkts[0];
924 mbuf1 = (uint64_t *)tx_pkts[1];
925 mbuf2 = (uint64_t *)tx_pkts[2];
926 mbuf3 = (uint64_t *)tx_pkts[3];
928 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
929 offsetof(struct rte_mbuf, buf_iova));
930 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
931 offsetof(struct rte_mbuf, buf_iova));
932 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
933 offsetof(struct rte_mbuf, buf_iova));
934 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
935 offsetof(struct rte_mbuf, buf_iova));
937 * Get mbuf's, olflags, iova, pktlen, dataoff
938 * dataoff_iovaX.D[0] = iova,
939 * dataoff_iovaX.D[1](15:0) = mbuf->dataoff
940 * len_olflagsX.D[0] = ol_flags,
941 * len_olflagsX.D[1](63:32) = mbuf->pkt_len
943 dataoff_iova0 = vld1q_u64(mbuf0);
944 len_olflags0 = vld1q_u64(mbuf0 + 2);
945 dataoff_iova1 = vld1q_u64(mbuf1);
946 len_olflags1 = vld1q_u64(mbuf1 + 2);
947 dataoff_iova2 = vld1q_u64(mbuf2);
948 len_olflags2 = vld1q_u64(mbuf2 + 2);
949 dataoff_iova3 = vld1q_u64(mbuf3);
950 len_olflags3 = vld1q_u64(mbuf3 + 2);
952 /* Move mbufs to point pool */
953 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
954 offsetof(struct rte_mbuf, pool) -
955 offsetof(struct rte_mbuf, buf_iova));
956 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
957 offsetof(struct rte_mbuf, pool) -
958 offsetof(struct rte_mbuf, buf_iova));
959 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
960 offsetof(struct rte_mbuf, pool) -
961 offsetof(struct rte_mbuf, buf_iova));
962 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
963 offsetof(struct rte_mbuf, pool) -
964 offsetof(struct rte_mbuf, buf_iova));
966 if (flags & (NIX_TX_OFFLOAD_OL3_OL4_CSUM_F |
967 NIX_TX_OFFLOAD_L3_L4_CSUM_F)) {
968 /* Get tx_offload for ol2, ol3, l2, l3 lengths */
970 * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7)
971 * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7)
974 asm volatile("LD1 {%[a].D}[0],[%[in]]\n\t"
975 : [a] "+w"(senddesc01_w1)
976 : [in] "r"(mbuf0 + 2)
979 asm volatile("LD1 {%[a].D}[1],[%[in]]\n\t"
980 : [a] "+w"(senddesc01_w1)
981 : [in] "r"(mbuf1 + 2)
984 asm volatile("LD1 {%[b].D}[0],[%[in]]\n\t"
985 : [b] "+w"(senddesc23_w1)
986 : [in] "r"(mbuf2 + 2)
989 asm volatile("LD1 {%[b].D}[1],[%[in]]\n\t"
990 : [b] "+w"(senddesc23_w1)
991 : [in] "r"(mbuf3 + 2)
994 /* Get pool pointer alone */
995 mbuf0 = (uint64_t *)*mbuf0;
996 mbuf1 = (uint64_t *)*mbuf1;
997 mbuf2 = (uint64_t *)*mbuf2;
998 mbuf3 = (uint64_t *)*mbuf3;
1000 /* Get pool pointer alone */
1001 mbuf0 = (uint64_t *)*mbuf0;
1002 mbuf1 = (uint64_t *)*mbuf1;
1003 mbuf2 = (uint64_t *)*mbuf2;
1004 mbuf3 = (uint64_t *)*mbuf3;
1007 const uint8x16_t shuf_mask2 = {
1008 0x4, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1009 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1011 xtmp128 = vzip2q_u64(len_olflags0, len_olflags1);
1012 ytmp128 = vzip2q_u64(len_olflags2, len_olflags3);
1014 /* Clear dataoff_iovaX.D[1] bits other than dataoff(15:0) */
1015 const uint64x2_t and_mask0 = {
1020 dataoff_iova0 = vandq_u64(dataoff_iova0, and_mask0);
1021 dataoff_iova1 = vandq_u64(dataoff_iova1, and_mask0);
1022 dataoff_iova2 = vandq_u64(dataoff_iova2, and_mask0);
1023 dataoff_iova3 = vandq_u64(dataoff_iova3, and_mask0);
1026 * Pick only 16 bits of pktlen preset at bits 63:32
1027 * and place them at bits 15:0.
1029 xtmp128 = vqtbl1q_u8(xtmp128, shuf_mask2);
1030 ytmp128 = vqtbl1q_u8(ytmp128, shuf_mask2);
1032 /* Add pairwise to get dataoff + iova in sgdesc_w1 */
1033 sgdesc01_w1 = vpaddq_u64(dataoff_iova0, dataoff_iova1);
1034 sgdesc23_w1 = vpaddq_u64(dataoff_iova2, dataoff_iova3);
1036 /* Orr both sgdesc_w0 and senddesc_w0 with 16 bits of
1037 * pktlen at 15:0 position.
1039 sgdesc01_w0 = vorrq_u64(sgdesc01_w0, xtmp128);
1040 sgdesc23_w0 = vorrq_u64(sgdesc23_w0, ytmp128);
1041 senddesc01_w0 = vorrq_u64(senddesc01_w0, xtmp128);
1042 senddesc23_w0 = vorrq_u64(senddesc23_w0, ytmp128);
1044 /* Move mbuf to point to pool_id. */
1045 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
1046 offsetof(struct rte_mempool, pool_id));
1047 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
1048 offsetof(struct rte_mempool, pool_id));
1049 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
1050 offsetof(struct rte_mempool, pool_id));
1051 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
1052 offsetof(struct rte_mempool, pool_id));
1054 if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
1055 !(flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
1057 * Lookup table to translate ol_flags to
1058 * il3/il4 types. But we still use ol3/ol4 types in
1059 * senddesc_w1 as only one header processing is enabled.
1061 const uint8x16_t tbl = {
1062 /* [0-15] = il4type:il3type */
1063 0x04, /* none (IPv6 assumed) */
1064 0x14, /* RTE_MBUF_F_TX_TCP_CKSUM (IPv6 assumed) */
1065 0x24, /* RTE_MBUF_F_TX_SCTP_CKSUM (IPv6 assumed) */
1066 0x34, /* RTE_MBUF_F_TX_UDP_CKSUM (IPv6 assumed) */
1067 0x03, /* RTE_MBUF_F_TX_IP_CKSUM */
1068 0x13, /* RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_TCP_CKSUM */
1069 0x23, /* RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_SCTP_CKSUM */
1070 0x33, /* RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_UDP_CKSUM */
1071 0x02, /* RTE_MBUF_F_TX_IPV4 */
1072 0x12, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_TCP_CKSUM */
1073 0x22, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_SCTP_CKSUM */
1074 0x32, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_UDP_CKSUM */
1075 0x03, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM */
1076 0x13, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM |
1077 * RTE_MBUF_F_TX_TCP_CKSUM
1079 0x23, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM |
1080 * RTE_MBUF_F_TX_SCTP_CKSUM
1082 0x33, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM |
1083 * RTE_MBUF_F_TX_UDP_CKSUM
1087 /* Extract olflags to translate to iltypes */
1088 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1089 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1092 * E(47):L3_LEN(9):L2_LEN(7+z)
1093 * E(47):L3_LEN(9):L2_LEN(7+z)
1095 senddesc01_w1 = vshlq_n_u64(senddesc01_w1, 1);
1096 senddesc23_w1 = vshlq_n_u64(senddesc23_w1, 1);
1098 /* Move OLFLAGS bits 55:52 to 51:48
1099 * with zeros preprended on the byte and rest
1102 xtmp128 = vshrq_n_u8(xtmp128, 4);
1103 ytmp128 = vshrq_n_u8(ytmp128, 4);
1105 * E(48):L3_LEN(8):L2_LEN(z+7)
1106 * E(48):L3_LEN(8):L2_LEN(z+7)
1108 const int8x16_t tshft3 = {
1109 -1, 0, 8, 8, 8, 8, 8, 8,
1110 -1, 0, 8, 8, 8, 8, 8, 8,
1113 senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
1114 senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
1117 ltypes01 = vqtbl1q_u8(tbl, xtmp128);
1118 ltypes23 = vqtbl1q_u8(tbl, ytmp128);
1120 /* Pick only relevant fields i.e Bit 48:55 of iltype
1121 * and place it in ol3/ol4type of senddesc_w1
1123 const uint8x16_t shuf_mask0 = {
1124 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0xFF, 0xFF, 0xFF,
1125 0xFF, 0xFF, 0xFF, 0xFF, 0xE, 0xFF, 0xFF, 0xFF,
1128 ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
1129 ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
1131 /* Prepare ol4ptr, ol3ptr from ol3len, ol2len.
1132 * a [E(32):E(16):OL3(8):OL2(8)]
1134 * a [E(32):E(16):(OL3+OL2):OL2]
1135 * => E(32):E(16)::OL4PTR(8):OL3PTR(8)
1137 senddesc01_w1 = vaddq_u8(senddesc01_w1,
1138 vshlq_n_u16(senddesc01_w1, 8));
1139 senddesc23_w1 = vaddq_u8(senddesc23_w1,
1140 vshlq_n_u16(senddesc23_w1, 8));
1142 /* Move ltypes to senddesc*_w1 */
1143 senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
1144 senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
1145 } else if (!(flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
1146 (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
1148 * Lookup table to translate ol_flags to
1152 const uint8x16_t tbl = {
1153 /* [0-15] = ol4type:ol3type */
1155 0x03, /* OUTER_IP_CKSUM */
1156 0x02, /* OUTER_IPV4 */
1157 0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */
1158 0x04, /* OUTER_IPV6 */
1159 0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */
1160 0x00, /* OUTER_IPV6 | OUTER_IPV4 */
1161 0x00, /* OUTER_IPV6 | OUTER_IPV4 |
1164 0x00, /* OUTER_UDP_CKSUM */
1165 0x33, /* OUTER_UDP_CKSUM | OUTER_IP_CKSUM */
1166 0x32, /* OUTER_UDP_CKSUM | OUTER_IPV4 */
1167 0x33, /* OUTER_UDP_CKSUM | OUTER_IPV4 |
1170 0x34, /* OUTER_UDP_CKSUM | OUTER_IPV6 */
1171 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1174 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1177 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1178 * OUTER_IPV4 | OUTER_IP_CKSUM
1182 /* Extract olflags to translate to iltypes */
1183 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1184 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1187 * E(47):OL3_LEN(9):OL2_LEN(7+z)
1188 * E(47):OL3_LEN(9):OL2_LEN(7+z)
1190 const uint8x16_t shuf_mask5 = {
1191 0x6, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1192 0xE, 0xD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1194 senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5);
1195 senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5);
1197 /* Extract outer ol flags only */
1198 const uint64x2_t o_cksum_mask = {
1203 xtmp128 = vandq_u64(xtmp128, o_cksum_mask);
1204 ytmp128 = vandq_u64(ytmp128, o_cksum_mask);
1206 /* Extract OUTER_UDP_CKSUM bit 41 and
1210 xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20);
1211 ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20);
1213 /* Shift oltype by 2 to start nibble from BIT(56)
1214 * instead of BIT(58)
1216 xtmp128 = vshrq_n_u8(xtmp128, 2);
1217 ytmp128 = vshrq_n_u8(ytmp128, 2);
1219 * E(48):L3_LEN(8):L2_LEN(z+7)
1220 * E(48):L3_LEN(8):L2_LEN(z+7)
1222 const int8x16_t tshft3 = {
1223 -1, 0, 8, 8, 8, 8, 8, 8,
1224 -1, 0, 8, 8, 8, 8, 8, 8,
1227 senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
1228 senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
1231 ltypes01 = vqtbl1q_u8(tbl, xtmp128);
1232 ltypes23 = vqtbl1q_u8(tbl, ytmp128);
1234 /* Pick only relevant fields i.e Bit 56:63 of oltype
1235 * and place it in ol3/ol4type of senddesc_w1
1237 const uint8x16_t shuf_mask0 = {
1238 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0xFF, 0xFF, 0xFF,
1239 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xFF, 0xFF, 0xFF,
1242 ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
1243 ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
1245 /* Prepare ol4ptr, ol3ptr from ol3len, ol2len.
1246 * a [E(32):E(16):OL3(8):OL2(8)]
1248 * a [E(32):E(16):(OL3+OL2):OL2]
1249 * => E(32):E(16)::OL4PTR(8):OL3PTR(8)
1251 senddesc01_w1 = vaddq_u8(senddesc01_w1,
1252 vshlq_n_u16(senddesc01_w1, 8));
1253 senddesc23_w1 = vaddq_u8(senddesc23_w1,
1254 vshlq_n_u16(senddesc23_w1, 8));
1256 /* Move ltypes to senddesc*_w1 */
1257 senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
1258 senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
1259 } else if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
1260 (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
1261 /* Lookup table to translate ol_flags to
1262 * ol4type, ol3type, il4type, il3type of senddesc_w1
1264 const uint8x16x2_t tbl = {{
1266 /* [0-15] = il4type:il3type */
1267 0x04, /* none (IPv6) */
1268 0x14, /* RTE_MBUF_F_TX_TCP_CKSUM (IPv6) */
1269 0x24, /* RTE_MBUF_F_TX_SCTP_CKSUM (IPv6) */
1270 0x34, /* RTE_MBUF_F_TX_UDP_CKSUM (IPv6) */
1271 0x03, /* RTE_MBUF_F_TX_IP_CKSUM */
1272 0x13, /* RTE_MBUF_F_TX_IP_CKSUM |
1273 * RTE_MBUF_F_TX_TCP_CKSUM
1275 0x23, /* RTE_MBUF_F_TX_IP_CKSUM |
1276 * RTE_MBUF_F_TX_SCTP_CKSUM
1278 0x33, /* RTE_MBUF_F_TX_IP_CKSUM |
1279 * RTE_MBUF_F_TX_UDP_CKSUM
1281 0x02, /* RTE_MBUF_F_TX_IPV4 */
1282 0x12, /* RTE_MBUF_F_TX_IPV4 |
1283 * RTE_MBUF_F_TX_TCP_CKSUM
1285 0x22, /* RTE_MBUF_F_TX_IPV4 |
1286 * RTE_MBUF_F_TX_SCTP_CKSUM
1288 0x32, /* RTE_MBUF_F_TX_IPV4 |
1289 * RTE_MBUF_F_TX_UDP_CKSUM
1291 0x03, /* RTE_MBUF_F_TX_IPV4 |
1292 * RTE_MBUF_F_TX_IP_CKSUM
1294 0x13, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM |
1295 * RTE_MBUF_F_TX_TCP_CKSUM
1297 0x23, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM |
1298 * RTE_MBUF_F_TX_SCTP_CKSUM
1300 0x33, /* RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM |
1301 * RTE_MBUF_F_TX_UDP_CKSUM
1306 /* [16-31] = ol4type:ol3type */
1308 0x03, /* OUTER_IP_CKSUM */
1309 0x02, /* OUTER_IPV4 */
1310 0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */
1311 0x04, /* OUTER_IPV6 */
1312 0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */
1313 0x00, /* OUTER_IPV6 | OUTER_IPV4 */
1314 0x00, /* OUTER_IPV6 | OUTER_IPV4 |
1317 0x00, /* OUTER_UDP_CKSUM */
1318 0x33, /* OUTER_UDP_CKSUM |
1321 0x32, /* OUTER_UDP_CKSUM |
1324 0x33, /* OUTER_UDP_CKSUM |
1325 * OUTER_IPV4 | OUTER_IP_CKSUM
1327 0x34, /* OUTER_UDP_CKSUM |
1330 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1333 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1336 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1337 * OUTER_IPV4 | OUTER_IP_CKSUM
1342 /* Extract olflags to translate to oltype & iltype */
1343 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1344 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1347 * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z)
1348 * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z)
1350 const uint32x4_t tshft_4 = {
1356 senddesc01_w1 = vshlq_u32(senddesc01_w1, tshft_4);
1357 senddesc23_w1 = vshlq_u32(senddesc23_w1, tshft_4);
1360 * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z)
1361 * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z)
1363 const uint8x16_t shuf_mask5 = {
1364 0x6, 0x5, 0x0, 0x1, 0xFF, 0xFF, 0xFF, 0xFF,
1365 0xE, 0xD, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF,
1367 senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5);
1368 senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5);
1370 /* Extract outer and inner header ol_flags */
1371 const uint64x2_t oi_cksum_mask = {
1376 xtmp128 = vandq_u64(xtmp128, oi_cksum_mask);
1377 ytmp128 = vandq_u64(ytmp128, oi_cksum_mask);
1379 /* Extract OUTER_UDP_CKSUM bit 41 and
1383 xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20);
1384 ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20);
1386 /* Shift right oltype by 2 and iltype by 4
1387 * to start oltype nibble from BIT(58)
1388 * instead of BIT(56) and iltype nibble from BIT(48)
1389 * instead of BIT(52).
1391 const int8x16_t tshft5 = {
1392 8, 8, 8, 8, 8, 8, -4, -2,
1393 8, 8, 8, 8, 8, 8, -4, -2,
1396 xtmp128 = vshlq_u8(xtmp128, tshft5);
1397 ytmp128 = vshlq_u8(ytmp128, tshft5);
1399 * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8)
1400 * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8)
1402 const int8x16_t tshft3 = {
1403 -1, 0, -1, 0, 0, 0, 0, 0,
1404 -1, 0, -1, 0, 0, 0, 0, 0,
1407 senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
1408 senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
1410 /* Mark Bit(4) of oltype */
1411 const uint64x2_t oi_cksum_mask2 = {
1416 xtmp128 = vorrq_u64(xtmp128, oi_cksum_mask2);
1417 ytmp128 = vorrq_u64(ytmp128, oi_cksum_mask2);
1420 ltypes01 = vqtbl2q_u8(tbl, xtmp128);
1421 ltypes23 = vqtbl2q_u8(tbl, ytmp128);
1423 /* Pick only relevant fields i.e Bit 48:55 of iltype and
1424 * Bit 56:63 of oltype and place it in corresponding
1425 * place in senddesc_w1.
1427 const uint8x16_t shuf_mask0 = {
1428 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0x6, 0xFF, 0xFF,
1429 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xE, 0xFF, 0xFF,
1432 ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
1433 ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
1435 /* Prepare l4ptr, l3ptr, ol4ptr, ol3ptr from
1436 * l3len, l2len, ol3len, ol2len.
1437 * a [E(32):L3(8):L2(8):OL3(8):OL2(8)]
1439 * a [E:(L3+L2):(L2+OL3):(OL3+OL2):OL2]
1441 * a [E:(L3+L2+OL3+OL2):(L2+OL3+OL2):(OL3+OL2):OL2]
1442 * => E(32):IL4PTR(8):IL3PTR(8):OL4PTR(8):OL3PTR(8)
1444 senddesc01_w1 = vaddq_u8(senddesc01_w1,
1445 vshlq_n_u32(senddesc01_w1, 8));
1446 senddesc23_w1 = vaddq_u8(senddesc23_w1,
1447 vshlq_n_u32(senddesc23_w1, 8));
1449 /* Continue preparing l4ptr, l3ptr, ol4ptr, ol3ptr */
1450 senddesc01_w1 = vaddq_u8(
1451 senddesc01_w1, vshlq_n_u32(senddesc01_w1, 16));
1452 senddesc23_w1 = vaddq_u8(
1453 senddesc23_w1, vshlq_n_u32(senddesc23_w1, 16));
1455 /* Move ltypes to senddesc*_w1 */
1456 senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
1457 senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
1460 xmask01 = vdupq_n_u64(0);
1462 asm volatile("LD1 {%[a].H}[0],[%[in]]\n\t"
1467 asm volatile("LD1 {%[a].H}[4],[%[in]]\n\t"
1472 asm volatile("LD1 {%[b].H}[0],[%[in]]\n\t"
1477 asm volatile("LD1 {%[b].H}[4],[%[in]]\n\t"
1481 xmask01 = vshlq_n_u64(xmask01, 20);
1482 xmask23 = vshlq_n_u64(xmask23, 20);
1484 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
1485 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
1487 if (flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
1488 /* Tx ol_flag for vlan. */
1489 const uint64x2_t olv = {RTE_MBUF_F_TX_VLAN, RTE_MBUF_F_TX_VLAN};
1490 /* Bit enable for VLAN1 */
1491 const uint64x2_t mlv = {BIT_ULL(49), BIT_ULL(49)};
1492 /* Tx ol_flag for QnQ. */
1493 const uint64x2_t olq = {RTE_MBUF_F_TX_QINQ, RTE_MBUF_F_TX_QINQ};
1494 /* Bit enable for VLAN0 */
1495 const uint64x2_t mlq = {BIT_ULL(48), BIT_ULL(48)};
1496 /* Load vlan values from packet. outer is VLAN 0 */
1497 uint64x2_t ext01 = {
1498 ((uint32_t)tx_pkts[0]->vlan_tci_outer) << 8 |
1499 ((uint64_t)tx_pkts[0]->vlan_tci) << 32,
1500 ((uint32_t)tx_pkts[1]->vlan_tci_outer) << 8 |
1501 ((uint64_t)tx_pkts[1]->vlan_tci) << 32,
1503 uint64x2_t ext23 = {
1504 ((uint32_t)tx_pkts[2]->vlan_tci_outer) << 8 |
1505 ((uint64_t)tx_pkts[2]->vlan_tci) << 32,
1506 ((uint32_t)tx_pkts[3]->vlan_tci_outer) << 8 |
1507 ((uint64_t)tx_pkts[3]->vlan_tci) << 32,
1510 /* Get ol_flags of the packets. */
1511 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1512 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1514 /* ORR vlan outer/inner values into cmd. */
1515 sendext01_w1 = vorrq_u64(sendext01_w1, ext01);
1516 sendext23_w1 = vorrq_u64(sendext23_w1, ext23);
1518 /* Test for offload enable bits and generate masks. */
1519 xtmp128 = vorrq_u64(vandq_u64(vtstq_u64(xtmp128, olv),
1521 vandq_u64(vtstq_u64(xtmp128, olq),
1523 ytmp128 = vorrq_u64(vandq_u64(vtstq_u64(ytmp128, olv),
1525 vandq_u64(vtstq_u64(ytmp128, olq),
1528 /* Set vlan enable bits into cmd based on mask. */
1529 sendext01_w1 = vorrq_u64(sendext01_w1, xtmp128);
1530 sendext23_w1 = vorrq_u64(sendext23_w1, ytmp128);
1533 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
1534 /* Tx ol_flag for timestam. */
1535 const uint64x2_t olf = {RTE_MBUF_F_TX_IEEE1588_TMST,
1536 RTE_MBUF_F_TX_IEEE1588_TMST};
1537 /* Set send mem alg to SUB. */
1538 const uint64x2_t alg = {BIT_ULL(59), BIT_ULL(59)};
1539 /* Increment send mem address by 8. */
1540 const uint64x2_t addr = {0x8, 0x8};
1542 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1543 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1545 /* Check if timestamp is requested and generate inverted
1546 * mask as we need not make any changes to default cmd
1549 xtmp128 = vmvnq_u32(vtstq_u64(olf, xtmp128));
1550 ytmp128 = vmvnq_u32(vtstq_u64(olf, ytmp128));
1552 /* Change send mem address to an 8 byte offset when
1553 * TSTMP is disabled.
1555 sendmem01_w1 = vaddq_u64(sendmem01_w1,
1556 vandq_u64(xtmp128, addr));
1557 sendmem23_w1 = vaddq_u64(sendmem23_w1,
1558 vandq_u64(ytmp128, addr));
1559 /* Change send mem alg to SUB when TSTMP is disabled. */
1560 sendmem01_w0 = vorrq_u64(sendmem01_w0,
1561 vandq_u64(xtmp128, alg));
1562 sendmem23_w0 = vorrq_u64(sendmem23_w0,
1563 vandq_u64(ytmp128, alg));
1565 cmd3[0] = vzip1q_u64(sendmem01_w0, sendmem01_w1);
1566 cmd3[1] = vzip2q_u64(sendmem01_w0, sendmem01_w1);
1567 cmd3[2] = vzip1q_u64(sendmem23_w0, sendmem23_w1);
1568 cmd3[3] = vzip2q_u64(sendmem23_w0, sendmem23_w1);
1571 if (flags & NIX_TX_OFFLOAD_TSO_F) {
1572 uint64_t sx_w0[NIX_DESCS_PER_LOOP];
1573 uint64_t sd_w1[NIX_DESCS_PER_LOOP];
1575 /* Extract SD W1 as we need to set L4 types. */
1576 vst1q_u64(sd_w1, senddesc01_w1);
1577 vst1q_u64(sd_w1 + 2, senddesc23_w1);
1579 /* Extract SX W0 as we need to set LSO fields. */
1580 vst1q_u64(sx_w0, sendext01_w0);
1581 vst1q_u64(sx_w0 + 2, sendext23_w0);
1583 /* Extract ol_flags. */
1584 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1585 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1587 /* Prepare individual mbufs. */
1588 cn9k_nix_prepare_tso(tx_pkts[0],
1589 (union nix_send_hdr_w1_u *)&sd_w1[0],
1590 (union nix_send_ext_w0_u *)&sx_w0[0],
1591 vgetq_lane_u64(xtmp128, 0), flags);
1593 cn9k_nix_prepare_tso(tx_pkts[1],
1594 (union nix_send_hdr_w1_u *)&sd_w1[1],
1595 (union nix_send_ext_w0_u *)&sx_w0[1],
1596 vgetq_lane_u64(xtmp128, 1), flags);
1598 cn9k_nix_prepare_tso(tx_pkts[2],
1599 (union nix_send_hdr_w1_u *)&sd_w1[2],
1600 (union nix_send_ext_w0_u *)&sx_w0[2],
1601 vgetq_lane_u64(ytmp128, 0), flags);
1603 cn9k_nix_prepare_tso(tx_pkts[3],
1604 (union nix_send_hdr_w1_u *)&sd_w1[3],
1605 (union nix_send_ext_w0_u *)&sx_w0[3],
1606 vgetq_lane_u64(ytmp128, 1), flags);
1608 senddesc01_w1 = vld1q_u64(sd_w1);
1609 senddesc23_w1 = vld1q_u64(sd_w1 + 2);
1611 sendext01_w0 = vld1q_u64(sx_w0);
1612 sendext23_w0 = vld1q_u64(sx_w0 + 2);
1615 if ((flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) &&
1616 !(flags & NIX_TX_MULTI_SEG_F)) {
1617 /* Set don't free bit if reference count > 1 */
1618 xmask01 = vdupq_n_u64(0);
1621 /* Move mbufs to iova */
1622 mbuf0 = (uint64_t *)tx_pkts[0];
1623 mbuf1 = (uint64_t *)tx_pkts[1];
1624 mbuf2 = (uint64_t *)tx_pkts[2];
1625 mbuf3 = (uint64_t *)tx_pkts[3];
1627 if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf0))
1628 vsetq_lane_u64(0x80000, xmask01, 0);
1630 RTE_MEMPOOL_CHECK_COOKIES(
1631 ((struct rte_mbuf *)mbuf0)->pool,
1632 (void **)&mbuf0, 1, 0);
1634 if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf1))
1635 vsetq_lane_u64(0x80000, xmask01, 1);
1637 RTE_MEMPOOL_CHECK_COOKIES(
1638 ((struct rte_mbuf *)mbuf1)->pool,
1639 (void **)&mbuf1, 1, 0);
1641 if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf2))
1642 vsetq_lane_u64(0x80000, xmask23, 0);
1644 RTE_MEMPOOL_CHECK_COOKIES(
1645 ((struct rte_mbuf *)mbuf2)->pool,
1646 (void **)&mbuf2, 1, 0);
1648 if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf3))
1649 vsetq_lane_u64(0x80000, xmask23, 1);
1651 RTE_MEMPOOL_CHECK_COOKIES(
1652 ((struct rte_mbuf *)mbuf3)->pool,
1653 (void **)&mbuf3, 1, 0);
1654 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
1655 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
1656 /* Ensuring mbuf fields which got updated in
1657 * cnxk_nix_prefree_seg are written before LMTST.
1660 } else if (!(flags & NIX_TX_MULTI_SEG_F)) {
1661 /* Move mbufs to iova */
1662 mbuf0 = (uint64_t *)tx_pkts[0];
1663 mbuf1 = (uint64_t *)tx_pkts[1];
1664 mbuf2 = (uint64_t *)tx_pkts[2];
1665 mbuf3 = (uint64_t *)tx_pkts[3];
1667 /* Mark mempool object as "put" since
1668 * it is freed by NIX
1670 RTE_MEMPOOL_CHECK_COOKIES(
1671 ((struct rte_mbuf *)mbuf0)->pool,
1672 (void **)&mbuf0, 1, 0);
1674 RTE_MEMPOOL_CHECK_COOKIES(
1675 ((struct rte_mbuf *)mbuf1)->pool,
1676 (void **)&mbuf1, 1, 0);
1678 RTE_MEMPOOL_CHECK_COOKIES(
1679 ((struct rte_mbuf *)mbuf2)->pool,
1680 (void **)&mbuf2, 1, 0);
1682 RTE_MEMPOOL_CHECK_COOKIES(
1683 ((struct rte_mbuf *)mbuf3)->pool,
1684 (void **)&mbuf3, 1, 0);
1685 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1690 /* Create 4W cmd for 4 mbufs (sendhdr, sgdesc) */
1691 cmd0[0] = vzip1q_u64(senddesc01_w0, senddesc01_w1);
1692 cmd0[1] = vzip2q_u64(senddesc01_w0, senddesc01_w1);
1693 cmd0[2] = vzip1q_u64(senddesc23_w0, senddesc23_w1);
1694 cmd0[3] = vzip2q_u64(senddesc23_w0, senddesc23_w1);
1696 cmd1[0] = vzip1q_u64(sgdesc01_w0, sgdesc01_w1);
1697 cmd1[1] = vzip2q_u64(sgdesc01_w0, sgdesc01_w1);
1698 cmd1[2] = vzip1q_u64(sgdesc23_w0, sgdesc23_w1);
1699 cmd1[3] = vzip2q_u64(sgdesc23_w0, sgdesc23_w1);
1701 if (flags & NIX_TX_NEED_EXT_HDR) {
1702 cmd2[0] = vzip1q_u64(sendext01_w0, sendext01_w1);
1703 cmd2[1] = vzip2q_u64(sendext01_w0, sendext01_w1);
1704 cmd2[2] = vzip1q_u64(sendext23_w0, sendext23_w1);
1705 cmd2[3] = vzip2q_u64(sendext23_w0, sendext23_w1);
1708 if (flags & NIX_TX_MULTI_SEG_F) {
1709 uint64_t seg_list[NIX_DESCS_PER_LOOP]
1710 [CNXK_NIX_TX_MSEG_SG_DWORDS - 2];
1711 uint8_t j, segdw[NIX_DESCS_PER_LOOP + 1];
1713 /* Build mseg list for each packet individually. */
1714 for (j = 0; j < NIX_DESCS_PER_LOOP; j++)
1715 segdw[j] = cn9k_nix_prepare_mseg_vec(tx_pkts[j],
1716 seg_list[j], &cmd0[j],
1720 /* Commit all changes to mbuf before LMTST. */
1721 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)
1724 cn9k_nix_xmit_pkts_mseg_vector(cmd0, cmd1, cmd2, cmd3,
1728 } else if (flags & NIX_TX_NEED_EXT_HDR) {
1729 /* With ext header in the command we can no longer send
1730 * all 4 packets together since LMTLINE is 128bytes.
1731 * Split and Tx twice.
1734 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
1735 vst1q_u64(lmt_addr, cmd0[0]);
1736 vst1q_u64(lmt_addr + 2, cmd2[0]);
1737 vst1q_u64(lmt_addr + 4, cmd1[0]);
1738 vst1q_u64(lmt_addr + 6, cmd3[0]);
1739 vst1q_u64(lmt_addr + 8, cmd0[1]);
1740 vst1q_u64(lmt_addr + 10, cmd2[1]);
1741 vst1q_u64(lmt_addr + 12, cmd1[1]);
1742 vst1q_u64(lmt_addr + 14, cmd3[1]);
1744 vst1q_u64(lmt_addr, cmd0[0]);
1745 vst1q_u64(lmt_addr + 2, cmd2[0]);
1746 vst1q_u64(lmt_addr + 4, cmd1[0]);
1747 vst1q_u64(lmt_addr + 6, cmd0[1]);
1748 vst1q_u64(lmt_addr + 8, cmd2[1]);
1749 vst1q_u64(lmt_addr + 10, cmd1[1]);
1751 lmt_status = roc_lmt_submit_ldeor(io_addr);
1752 } while (lmt_status == 0);
1755 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
1756 vst1q_u64(lmt_addr, cmd0[2]);
1757 vst1q_u64(lmt_addr + 2, cmd2[2]);
1758 vst1q_u64(lmt_addr + 4, cmd1[2]);
1759 vst1q_u64(lmt_addr + 6, cmd3[2]);
1760 vst1q_u64(lmt_addr + 8, cmd0[3]);
1761 vst1q_u64(lmt_addr + 10, cmd2[3]);
1762 vst1q_u64(lmt_addr + 12, cmd1[3]);
1763 vst1q_u64(lmt_addr + 14, cmd3[3]);
1765 vst1q_u64(lmt_addr, cmd0[2]);
1766 vst1q_u64(lmt_addr + 2, cmd2[2]);
1767 vst1q_u64(lmt_addr + 4, cmd1[2]);
1768 vst1q_u64(lmt_addr + 6, cmd0[3]);
1769 vst1q_u64(lmt_addr + 8, cmd2[3]);
1770 vst1q_u64(lmt_addr + 10, cmd1[3]);
1772 lmt_status = roc_lmt_submit_ldeor(io_addr);
1773 } while (lmt_status == 0);
1776 vst1q_u64(lmt_addr, cmd0[0]);
1777 vst1q_u64(lmt_addr + 2, cmd1[0]);
1778 vst1q_u64(lmt_addr + 4, cmd0[1]);
1779 vst1q_u64(lmt_addr + 6, cmd1[1]);
1780 vst1q_u64(lmt_addr + 8, cmd0[2]);
1781 vst1q_u64(lmt_addr + 10, cmd1[2]);
1782 vst1q_u64(lmt_addr + 12, cmd0[3]);
1783 vst1q_u64(lmt_addr + 14, cmd1[3]);
1784 lmt_status = roc_lmt_submit_ldeor(io_addr);
1785 } while (lmt_status == 0);
1787 tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP;
1790 if (unlikely(pkts_left)) {
1791 if (flags & NIX_TX_MULTI_SEG_F)
1792 pkts += cn9k_nix_xmit_pkts_mseg(tx_queue, tx_pkts,
1793 pkts_left, cmd, flags);
1795 pkts += cn9k_nix_xmit_pkts(tx_queue, tx_pkts, pkts_left,
1803 static __rte_always_inline uint16_t
1804 cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
1805 uint16_t pkts, uint64_t *cmd, const uint16_t flags)
1807 RTE_SET_USED(tx_queue);
1808 RTE_SET_USED(tx_pkts);
1811 RTE_SET_USED(flags);
1816 #define L3L4CSUM_F NIX_TX_OFFLOAD_L3_L4_CSUM_F
1817 #define OL3OL4CSUM_F NIX_TX_OFFLOAD_OL3_OL4_CSUM_F
1818 #define VLAN_F NIX_TX_OFFLOAD_VLAN_QINQ_F
1819 #define NOFF_F NIX_TX_OFFLOAD_MBUF_NOFF_F
1820 #define TSO_F NIX_TX_OFFLOAD_TSO_F
1821 #define TSP_F NIX_TX_OFFLOAD_TSTAMP_F
1822 #define T_SEC_F NIX_TX_OFFLOAD_SECURITY_F
1824 /* [T_SEC_F] [TSP] [TSO] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */
1825 #define NIX_TX_FASTPATH_MODES \
1826 T(no_offload, 0, 0, 0, 0, 0, 0, 0, 4, \
1827 NIX_TX_OFFLOAD_NONE) \
1828 T(l3l4csum, 0, 0, 0, 0, 0, 0, 1, 4, \
1830 T(ol3ol4csum, 0, 0, 0, 0, 0, 1, 0, 4, \
1832 T(ol3ol4csum_l3l4csum, 0, 0, 0, 0, 0, 1, 1, 4, \
1833 OL3OL4CSUM_F | L3L4CSUM_F) \
1834 T(vlan, 0, 0, 0, 0, 1, 0, 0, 6, \
1836 T(vlan_l3l4csum, 0, 0, 0, 0, 1, 0, 1, 6, \
1837 VLAN_F | L3L4CSUM_F) \
1838 T(vlan_ol3ol4csum, 0, 0, 0, 0, 1, 1, 0, 6, \
1839 VLAN_F | OL3OL4CSUM_F) \
1840 T(vlan_ol3ol4csum_l3l4csum, 0, 0, 0, 0, 1, 1, 1, 6, \
1841 VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1842 T(noff, 0, 0, 0, 1, 0, 0, 0, 4, \
1844 T(noff_l3l4csum, 0, 0, 0, 1, 0, 0, 1, 4, \
1845 NOFF_F | L3L4CSUM_F) \
1846 T(noff_ol3ol4csum, 0, 0, 0, 1, 0, 1, 0, 4, \
1847 NOFF_F | OL3OL4CSUM_F) \
1848 T(noff_ol3ol4csum_l3l4csum, 0, 0, 0, 1, 0, 1, 1, 4, \
1849 NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1850 T(noff_vlan, 0, 0, 0, 1, 1, 0, 0, 6, \
1852 T(noff_vlan_l3l4csum, 0, 0, 0, 1, 1, 0, 1, 6, \
1853 NOFF_F | VLAN_F | L3L4CSUM_F) \
1854 T(noff_vlan_ol3ol4csum, 0, 0, 0, 1, 1, 1, 0, 6, \
1855 NOFF_F | VLAN_F | OL3OL4CSUM_F) \
1856 T(noff_vlan_ol3ol4csum_l3l4csum, 0, 0, 0, 1, 1, 1, 1, 6, \
1857 NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1858 T(tso, 0, 0, 1, 0, 0, 0, 0, 6, \
1860 T(tso_l3l4csum, 0, 0, 1, 0, 0, 0, 1, 6, \
1861 TSO_F | L3L4CSUM_F) \
1862 T(tso_ol3ol4csum, 0, 0, 1, 0, 0, 1, 0, 6, \
1863 TSO_F | OL3OL4CSUM_F) \
1864 T(tso_ol3ol4csum_l3l4csum, 0, 0, 1, 0, 0, 1, 1, 6, \
1865 TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1866 T(tso_vlan, 0, 0, 1, 0, 1, 0, 0, 6, \
1868 T(tso_vlan_l3l4csum, 0, 0, 1, 0, 1, 0, 1, 6, \
1869 TSO_F | VLAN_F | L3L4CSUM_F) \
1870 T(tso_vlan_ol3ol4csum, 0, 0, 1, 0, 1, 1, 0, 6, \
1871 TSO_F | VLAN_F | OL3OL4CSUM_F) \
1872 T(tso_vlan_ol3ol4csum_l3l4csum, 0, 0, 1, 0, 1, 1, 1, 6, \
1873 TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1874 T(tso_noff, 0, 0, 1, 1, 0, 0, 0, 6, \
1876 T(tso_noff_l3l4csum, 0, 0, 1, 1, 0, 0, 1, 6, \
1877 TSO_F | NOFF_F | L3L4CSUM_F) \
1878 T(tso_noff_ol3ol4csum, 0, 0, 1, 1, 0, 1, 0, 6, \
1879 TSO_F | NOFF_F | OL3OL4CSUM_F) \
1880 T(tso_noff_ol3ol4csum_l3l4csum, 0, 0, 1, 1, 0, 1, 1, 6, \
1881 TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1882 T(tso_noff_vlan, 0, 0, 1, 1, 1, 0, 0, 6, \
1883 TSO_F | NOFF_F | VLAN_F) \
1884 T(tso_noff_vlan_l3l4csum, 0, 0, 1, 1, 1, 0, 1, 6, \
1885 TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
1886 T(tso_noff_vlan_ol3ol4csum, 0, 0, 1, 1, 1, 1, 0, 6, \
1887 TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
1888 T(tso_noff_vlan_ol3ol4csum_l3l4csum, 0, 0, 1, 1, 1, 1, 1, 6, \
1889 TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1890 T(ts, 0, 1, 0, 0, 0, 0, 0, 8, \
1892 T(ts_l3l4csum, 0, 1, 0, 0, 0, 0, 1, 8, \
1893 TSP_F | L3L4CSUM_F) \
1894 T(ts_ol3ol4csum, 0, 1, 0, 0, 0, 1, 0, 8, \
1895 TSP_F | OL3OL4CSUM_F) \
1896 T(ts_ol3ol4csum_l3l4csum, 0, 1, 0, 0, 0, 1, 1, 8, \
1897 TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1898 T(ts_vlan, 0, 1, 0, 0, 1, 0, 0, 8, \
1900 T(ts_vlan_l3l4csum, 0, 1, 0, 0, 1, 0, 1, 8, \
1901 TSP_F | VLAN_F | L3L4CSUM_F) \
1902 T(ts_vlan_ol3ol4csum, 0, 1, 0, 0, 1, 1, 0, 8, \
1903 TSP_F | VLAN_F | OL3OL4CSUM_F) \
1904 T(ts_vlan_ol3ol4csum_l3l4csum, 0, 1, 0, 0, 1, 1, 1, 8, \
1905 TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1906 T(ts_noff, 0, 1, 0, 1, 0, 0, 0, 8, \
1908 T(ts_noff_l3l4csum, 0, 1, 0, 1, 0, 0, 1, 8, \
1909 TSP_F | NOFF_F | L3L4CSUM_F) \
1910 T(ts_noff_ol3ol4csum, 0, 1, 0, 1, 0, 1, 0, 8, \
1911 TSP_F | NOFF_F | OL3OL4CSUM_F) \
1912 T(ts_noff_ol3ol4csum_l3l4csum, 0, 1, 0, 1, 0, 1, 1, 8, \
1913 TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1914 T(ts_noff_vlan, 0, 1, 0, 1, 1, 0, 0, 8, \
1915 TSP_F | NOFF_F | VLAN_F) \
1916 T(ts_noff_vlan_l3l4csum, 0, 1, 0, 1, 1, 0, 1, 8, \
1917 TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
1918 T(ts_noff_vlan_ol3ol4csum, 0, 1, 0, 1, 1, 1, 0, 8, \
1919 TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
1920 T(ts_noff_vlan_ol3ol4csum_l3l4csum, 0, 1, 0, 1, 1, 1, 1, 8, \
1921 TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1922 T(ts_tso, 0, 1, 1, 0, 0, 0, 0, 8, \
1924 T(ts_tso_l3l4csum, 0, 1, 1, 0, 0, 0, 1, 8, \
1925 TSP_F | TSO_F | L3L4CSUM_F) \
1926 T(ts_tso_ol3ol4csum, 0, 1, 1, 0, 0, 1, 0, 8, \
1927 TSP_F | TSO_F | OL3OL4CSUM_F) \
1928 T(ts_tso_ol3ol4csum_l3l4csum, 0, 1, 1, 0, 0, 1, 1, 8, \
1929 TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1930 T(ts_tso_vlan, 0, 1, 1, 0, 1, 0, 0, 8, \
1931 TSP_F | TSO_F | VLAN_F) \
1932 T(ts_tso_vlan_l3l4csum, 0, 1, 1, 0, 1, 0, 1, 8, \
1933 TSP_F | TSO_F | VLAN_F | L3L4CSUM_F) \
1934 T(ts_tso_vlan_ol3ol4csum, 0, 1, 1, 0, 1, 1, 0, 8, \
1935 TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \
1936 T(ts_tso_vlan_ol3ol4csum_l3l4csum, 0, 1, 1, 0, 1, 1, 1, 8, \
1937 TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1938 T(ts_tso_noff, 0, 1, 1, 1, 0, 0, 0, 8, \
1939 TSP_F | TSO_F | NOFF_F) \
1940 T(ts_tso_noff_l3l4csum, 0, 1, 1, 1, 0, 0, 1, 8, \
1941 TSP_F | TSO_F | NOFF_F | L3L4CSUM_F) \
1942 T(ts_tso_noff_ol3ol4csum, 0, 1, 1, 1, 0, 1, 0, 8, \
1943 TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \
1944 T(ts_tso_noff_ol3ol4csum_l3l4csum, 0, 1, 1, 1, 0, 1, 1, 8, \
1945 TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1946 T(ts_tso_noff_vlan, 0, 1, 1, 1, 1, 0, 0, 8, \
1947 TSP_F | TSO_F | NOFF_F | VLAN_F) \
1948 T(ts_tso_noff_vlan_l3l4csum, 0, 1, 1, 1, 1, 0, 1, 8, \
1949 TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
1950 T(ts_tso_noff_vlan_ol3ol4csum, 0, 1, 1, 1, 1, 1, 0, 8, \
1951 TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
1952 T(ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 0, 1, 1, 1, 1, 1, 1, 8, \
1953 TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)\
1954 T(sec, 1, 0, 0, 0, 0, 0, 0, 4, \
1956 T(sec_l3l4csum, 1, 0, 0, 0, 0, 0, 1, 4, \
1957 T_SEC_F | L3L4CSUM_F) \
1958 T(sec_ol3ol4csum, 1, 0, 0, 0, 0, 1, 0, 4, \
1959 T_SEC_F | OL3OL4CSUM_F) \
1960 T(sec_ol3ol4csum_l3l4csum, 1, 0, 0, 0, 0, 1, 1, 4, \
1961 T_SEC_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1962 T(sec_vlan, 1, 0, 0, 0, 1, 0, 0, 6, \
1964 T(sec_vlan_l3l4csum, 1, 0, 0, 0, 1, 0, 1, 6, \
1965 T_SEC_F | VLAN_F | L3L4CSUM_F) \
1966 T(sec_vlan_ol3ol4csum, 1, 0, 0, 0, 1, 1, 0, 6, \
1967 T_SEC_F | VLAN_F | OL3OL4CSUM_F) \
1968 T(sec_vlan_ol3ol4csum_l3l4csum, 1, 0, 0, 0, 1, 1, 1, 6, \
1969 T_SEC_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1970 T(sec_noff, 1, 0, 0, 1, 0, 0, 0, 4, \
1972 T(sec_noff_l3l4csum, 1, 0, 0, 1, 0, 0, 1, 4, \
1973 T_SEC_F | NOFF_F | L3L4CSUM_F) \
1974 T(sec_noff_ol3ol4csum, 1, 0, 0, 1, 0, 1, 0, 4, \
1975 T_SEC_F | NOFF_F | OL3OL4CSUM_F) \
1976 T(sec_noff_ol3ol4csum_l3l4csum, 1, 0, 0, 1, 0, 1, 1, 4, \
1977 T_SEC_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1978 T(sec_noff_vlan, 1, 0, 0, 1, 1, 0, 0, 6, \
1979 T_SEC_F | NOFF_F | VLAN_F) \
1980 T(sec_noff_vlan_l3l4csum, 1, 0, 0, 1, 1, 0, 1, 6, \
1981 T_SEC_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
1982 T(sec_noff_vlan_ol3ol4csum, 1, 0, 0, 1, 1, 1, 0, 6, \
1983 T_SEC_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
1984 T(sec_noff_vlan_ol3ol4csum_l3l4csum, 1, 0, 0, 1, 1, 1, 1, 6, \
1985 T_SEC_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1986 T(sec_tso, 1, 0, 1, 0, 0, 0, 0, 6, \
1988 T(sec_tso_l3l4csum, 1, 0, 1, 0, 0, 0, 1, 6, \
1989 T_SEC_F | TSO_F | L3L4CSUM_F) \
1990 T(sec_tso_ol3ol4csum, 1, 0, 1, 0, 0, 1, 0, 6, \
1991 T_SEC_F | TSO_F | OL3OL4CSUM_F) \
1992 T(sec_tso_ol3ol4csum_l3l4csum, 1, 0, 1, 0, 0, 1, 1, 6, \
1993 T_SEC_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \
1994 T(sec_tso_vlan, 1, 0, 1, 0, 1, 0, 0, 6, \
1995 T_SEC_F | TSO_F | VLAN_F) \
1996 T(sec_tso_vlan_l3l4csum, 1, 0, 1, 0, 1, 0, 1, 6, \
1997 T_SEC_F | TSO_F | VLAN_F | L3L4CSUM_F) \
1998 T(sec_tso_vlan_ol3ol4csum, 1, 0, 1, 0, 1, 1, 0, 6, \
1999 T_SEC_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \
2000 T(sec_tso_vlan_ol3ol4csum_l3l4csum, 1, 0, 1, 0, 1, 1, 1, 6, \
2001 T_SEC_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2002 T(sec_tso_noff, 1, 0, 1, 1, 0, 0, 0, 6, \
2003 T_SEC_F | TSO_F | NOFF_F) \
2004 T(sec_tso_noff_l3l4csum, 1, 0, 1, 1, 0, 0, 1, 6, \
2005 T_SEC_F | TSO_F | NOFF_F | L3L4CSUM_F) \
2006 T(sec_tso_noff_ol3ol4csum, 1, 0, 1, 1, 0, 1, 0, 6, \
2007 T_SEC_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \
2008 T(sec_tso_noff_ol3ol4csum_l3l4csum, 1, 0, 1, 1, 0, 1, 1, 6, \
2009 T_SEC_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2010 T(sec_tso_noff_vlan, 1, 0, 1, 1, 1, 0, 0, 6, \
2011 T_SEC_F | TSO_F | NOFF_F | VLAN_F) \
2012 T(sec_tso_noff_vlan_l3l4csum, 1, 0, 1, 1, 1, 0, 1, 6, \
2013 T_SEC_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
2014 T(sec_tso_noff_vlan_ol3ol4csum, 1, 0, 1, 1, 1, 1, 0, 6, \
2015 T_SEC_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
2016 T(sec_tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 0, 1, 1, 1, 1, 1, 6, \
2017 T_SEC_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)\
2018 T(sec_ts, 1, 1, 0, 0, 0, 0, 0, 8, \
2020 T(sec_ts_l3l4csum, 1, 1, 0, 0, 0, 0, 1, 8, \
2021 T_SEC_F | TSP_F | L3L4CSUM_F) \
2022 T(sec_ts_ol3ol4csum, 1, 1, 0, 0, 0, 1, 0, 8, \
2023 T_SEC_F | TSP_F | OL3OL4CSUM_F) \
2024 T(sec_ts_ol3ol4csum_l3l4csum, 1, 1, 0, 0, 0, 1, 1, 8, \
2025 T_SEC_F | TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2026 T(sec_ts_vlan, 1, 1, 0, 0, 1, 0, 0, 8, \
2027 T_SEC_F | TSP_F | VLAN_F) \
2028 T(sec_ts_vlan_l3l4csum, 1, 1, 0, 0, 1, 0, 1, 8, \
2029 T_SEC_F | TSP_F | VLAN_F | L3L4CSUM_F) \
2030 T(sec_ts_vlan_ol3ol4csum, 1, 1, 0, 0, 1, 1, 0, 8, \
2031 T_SEC_F | TSP_F | VLAN_F | OL3OL4CSUM_F) \
2032 T(sec_ts_vlan_ol3ol4csum_l3l4csum, 1, 1, 0, 0, 1, 1, 1, 8, \
2033 T_SEC_F | TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2034 T(sec_ts_noff, 1, 1, 0, 1, 0, 0, 0, 8, \
2035 T_SEC_F | TSP_F | NOFF_F) \
2036 T(sec_ts_noff_l3l4csum, 1, 1, 0, 1, 0, 0, 1, 8, \
2037 T_SEC_F | TSP_F | NOFF_F | L3L4CSUM_F) \
2038 T(sec_ts_noff_ol3ol4csum, 1, 1, 0, 1, 0, 1, 0, 8, \
2039 T_SEC_F | TSP_F | NOFF_F | OL3OL4CSUM_F) \
2040 T(sec_ts_noff_ol3ol4csum_l3l4csum, 1, 1, 0, 1, 0, 1, 1, 8, \
2041 T_SEC_F | TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2042 T(sec_ts_noff_vlan, 1, 1, 0, 1, 1, 0, 0, 8, \
2043 T_SEC_F | TSP_F | NOFF_F | VLAN_F) \
2044 T(sec_ts_noff_vlan_l3l4csum, 1, 1, 0, 1, 1, 0, 1, 8, \
2045 T_SEC_F | TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
2046 T(sec_ts_noff_vlan_ol3ol4csum, 1, 1, 0, 1, 1, 1, 0, 8, \
2047 T_SEC_F | TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \
2048 T(sec_ts_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 0, 1, 1, 1, 1, 8, \
2049 T_SEC_F | TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)\
2050 T(sec_ts_tso, 1, 1, 1, 0, 0, 0, 0, 8, \
2051 T_SEC_F | TSP_F | TSO_F) \
2052 T(sec_ts_tso_l3l4csum, 1, 1, 1, 0, 0, 0, 1, 8, \
2053 T_SEC_F | TSP_F | TSO_F | L3L4CSUM_F) \
2054 T(sec_ts_tso_ol3ol4csum, 1, 1, 1, 0, 0, 1, 0, 8, \
2055 T_SEC_F | TSP_F | TSO_F | OL3OL4CSUM_F) \
2056 T(sec_ts_tso_ol3ol4csum_l3l4csum, 1, 1, 1, 0, 0, 1, 1, 8, \
2057 T_SEC_F | TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2058 T(sec_ts_tso_vlan, 1, 1, 1, 0, 1, 0, 0, 8, \
2059 T_SEC_F | TSP_F | TSO_F | VLAN_F) \
2060 T(sec_ts_tso_vlan_l3l4csum, 1, 1, 1, 0, 1, 0, 1, 8, \
2061 T_SEC_F | TSP_F | TSO_F | VLAN_F | L3L4CSUM_F) \
2062 T(sec_ts_tso_vlan_ol3ol4csum, 1, 1, 1, 0, 1, 1, 0, 8, \
2063 T_SEC_F | TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \
2064 T(sec_ts_tso_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 0, 1, 1, 1, 8, \
2065 T_SEC_F | TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2066 T(sec_ts_tso_noff, 1, 1, 1, 1, 0, 0, 0, 8, \
2067 T_SEC_F | TSP_F | TSO_F | NOFF_F) \
2068 T(sec_ts_tso_noff_l3l4csum, 1, 1, 1, 1, 0, 0, 1, 8, \
2069 T_SEC_F | TSP_F | TSO_F | NOFF_F | L3L4CSUM_F) \
2070 T(sec_ts_tso_noff_ol3ol4csum, 1, 1, 1, 1, 0, 1, 0, 8, \
2071 T_SEC_F | TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \
2072 T(sec_ts_tso_noff_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 0, 1, 1, 8, \
2073 T_SEC_F | TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \
2074 T(sec_ts_tso_noff_vlan, 1, 1, 1, 1, 1, 0, 0, 8, \
2075 T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F) \
2076 T(sec_ts_tso_noff_vlan_l3l4csum, 1, 1, 1, 1, 1, 0, 1, 8, \
2077 T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \
2078 T(sec_ts_tso_noff_vlan_ol3ol4csum, 1, 1, 1, 1, 1, 1, 0, 8, \
2079 T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F)\
2080 T(sec_ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 1, 1, 1, 8, \
2081 T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | \
2084 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
2085 uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_##name( \
2086 void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \
2088 uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_mseg_##name( \
2089 void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \
2091 uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_vec_##name( \
2092 void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \
2094 uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_vec_mseg_##name( \
2095 void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts);
2097 NIX_TX_FASTPATH_MODES
2100 #endif /* __CN9K_TX_H__ */