drivers/net/cnxk/cn10k_tx.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4 #ifndef __CN10K_TX_H__
   5 #define __CN10K_TX_H__
   6
   7 #include <rte_vect.h>
   8
   9 #define NIX_TX_OFFLOAD_NONE           (0)
  10 #define NIX_TX_OFFLOAD_L3_L4_CSUM_F   BIT(0)
  11 #define NIX_TX_OFFLOAD_OL3_OL4_CSUM_F BIT(1)
  12 #define NIX_TX_OFFLOAD_VLAN_QINQ_F    BIT(2)
  13 #define NIX_TX_OFFLOAD_MBUF_NOFF_F    BIT(3)
  14 #define NIX_TX_OFFLOAD_TSO_F          BIT(4)
  15 #define NIX_TX_OFFLOAD_TSTAMP_F       BIT(5)
  16
  17 /* Flags to control xmit_prepare function.
  18  * Defining it from backwards to denote its been
  19  * not used as offload flags to pick function
  20  */
  21 #define NIX_TX_MULTI_SEG_F BIT(15)
  22
  23 #define NIX_TX_NEED_SEND_HDR_W1                                                \
  24         (NIX_TX_OFFLOAD_L3_L4_CSUM_F | NIX_TX_OFFLOAD_OL3_OL4_CSUM_F |         \
  25          NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F)
  26
  27 #define NIX_TX_NEED_EXT_HDR                                                    \
  28         (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSTAMP_F |                \
  29          NIX_TX_OFFLOAD_TSO_F)
  30
  31 #define NIX_XMIT_FC_OR_RETURN(txq, pkts)                                       \
  32         do {                                                                   \
  33                 /* Cached value is low, Update the fc_cache_pkts */            \
  34                 if (unlikely((txq)->fc_cache_pkts < (pkts))) {                 \
  35                         /* Multiply with sqe_per_sqb to express in pkts */     \
  36                         (txq)->fc_cache_pkts =                                 \
  37                                 ((txq)->nb_sqb_bufs_adj - *(txq)->fc_mem)      \
  38                                 << (txq)->sqes_per_sqb_log2;                   \
  39                         /* Check it again for the room */                      \
  40                         if (unlikely((txq)->fc_cache_pkts < (pkts)))           \
  41                                 return 0;                                      \
  42                 }                                                              \
  43         } while (0)
  44
  45 #define LMT_OFF(lmt_addr, lmt_num, offset)                                     \
  46         (void *)((lmt_addr) + ((lmt_num) << ROC_LMT_LINE_SIZE_LOG2) + (offset))
  47
  48 /* Function to determine no of tx subdesc required in case ext
  49  * sub desc is enabled.
  50  */
  51 static __rte_always_inline int
  52 cn10k_nix_tx_ext_subs(const uint16_t flags)
  53 {
  54         return (flags & NIX_TX_OFFLOAD_TSTAMP_F)
  55                        ? 2
  56                        : ((flags &
  57                            (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F))
  58                                   ? 1
  59                                   : 0);
  60 }
  61
  62 static __rte_always_inline uint8_t
  63 cn10k_nix_pkts_per_vec_brst(const uint16_t flags)
  64 {
  65         return ((flags & NIX_TX_NEED_EXT_HDR) ? 2 : 4)
  66                << ROC_LMT_LINES_PER_CORE_LOG2;
  67 }
  68
  69 static __rte_always_inline uint8_t
  70 cn10k_nix_tx_dwords_per_line(const uint16_t flags)
  71 {
  72         return (flags & NIX_TX_NEED_EXT_HDR) ?
  73                              ((flags & NIX_TX_OFFLOAD_TSTAMP_F) ? 8 : 6) :
  74                              8;
  75 }
  76
  77 static __rte_always_inline uint64_t
  78 cn10k_nix_tx_steor_data(const uint16_t flags)
  79 {
  80         const uint64_t dw_m1 = cn10k_nix_tx_ext_subs(flags) + 1;
  81         uint64_t data;
  82
  83         /* This will be moved to addr area */
  84         data = dw_m1;
  85         /* 15 vector sizes for single seg */
  86         data |= dw_m1 << 19;
  87         data |= dw_m1 << 22;
  88         data |= dw_m1 << 25;
  89         data |= dw_m1 << 28;
  90         data |= dw_m1 << 31;
  91         data |= dw_m1 << 34;
  92         data |= dw_m1 << 37;
  93         data |= dw_m1 << 40;
  94         data |= dw_m1 << 43;
  95         data |= dw_m1 << 46;
  96         data |= dw_m1 << 49;
  97         data |= dw_m1 << 52;
  98         data |= dw_m1 << 55;
  99         data |= dw_m1 << 58;
 100         data |= dw_m1 << 61;
 101
 102         return data;
 103 }
 104
 105 static __rte_always_inline uint64_t
 106 cn10k_nix_tx_steor_vec_data(const uint16_t flags)
 107 {
 108         const uint64_t dw_m1 = cn10k_nix_tx_dwords_per_line(flags) - 1;
 109         uint64_t data;
 110
 111         /* This will be moved to addr area */
 112         data = dw_m1;
 113         /* 15 vector sizes for single seg */
 114         data |= dw_m1 << 19;
 115         data |= dw_m1 << 22;
 116         data |= dw_m1 << 25;
 117         data |= dw_m1 << 28;
 118         data |= dw_m1 << 31;
 119         data |= dw_m1 << 34;
 120         data |= dw_m1 << 37;
 121         data |= dw_m1 << 40;
 122         data |= dw_m1 << 43;
 123         data |= dw_m1 << 46;
 124         data |= dw_m1 << 49;
 125         data |= dw_m1 << 52;
 126         data |= dw_m1 << 55;
 127         data |= dw_m1 << 58;
 128         data |= dw_m1 << 61;
 129
 130         return data;
 131 }
 132
 133 static __rte_always_inline void
 134 cn10k_nix_tx_skeleton(const struct cn10k_eth_txq *txq, uint64_t *cmd,
 135                       const uint16_t flags)
 136 {
 137         /* Send hdr */
 138         cmd[0] = txq->send_hdr_w0;
 139         cmd[1] = 0;
 140         cmd += 2;
 141
 142         /* Send ext if present */
 143         if (flags & NIX_TX_NEED_EXT_HDR) {
 144                 *(__uint128_t *)cmd = *(const __uint128_t *)txq->cmd;
 145                 cmd += 2;
 146         }
 147
 148         /* Send sg */
 149         cmd[0] = txq->sg_w0;
 150         cmd[1] = 0;
 151 }
 152
 153 static __rte_always_inline void
 154 cn10k_nix_xmit_prepare_tso(struct rte_mbuf *m, const uint64_t flags)
 155 {
 156         uint64_t mask, ol_flags = m->ol_flags;
 157
 158         if (flags & NIX_TX_OFFLOAD_TSO_F && (ol_flags & PKT_TX_TCP_SEG)) {
 159                 uintptr_t mdata = rte_pktmbuf_mtod(m, uintptr_t);
 160                 uint16_t *iplen, *oiplen, *oudplen;
 161                 uint16_t lso_sb, paylen;
 162
 163                 mask = -!!(ol_flags & (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IPV6));
 164                 lso_sb = (mask & (m->outer_l2_len + m->outer_l3_len)) +
 165                          m->l2_len + m->l3_len + m->l4_len;
 166
 167                 /* Reduce payload len from base headers */
 168                 paylen = m->pkt_len - lso_sb;
 169
 170                 /* Get iplen position assuming no tunnel hdr */
 171                 iplen = (uint16_t *)(mdata + m->l2_len +
 172                                      (2 << !!(ol_flags & PKT_TX_IPV6)));
 173                 /* Handle tunnel tso */
 174                 if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
 175                     (ol_flags & PKT_TX_TUNNEL_MASK)) {
 176                         const uint8_t is_udp_tun =
 177                                 (CNXK_NIX_UDP_TUN_BITMASK >>
 178                                  ((ol_flags & PKT_TX_TUNNEL_MASK) >> 45)) &
 179                                 0x1;
 180
 181                         oiplen = (uint16_t *)(mdata + m->outer_l2_len +
 182                                               (2 << !!(ol_flags &
 183                                                        PKT_TX_OUTER_IPV6)));
 184                         *oiplen = rte_cpu_to_be_16(rte_be_to_cpu_16(*oiplen) -
 185                                                    paylen);
 186
 187                         /* Update format for UDP tunneled packet */
 188                         if (is_udp_tun) {
 189                                 oudplen = (uint16_t *)(mdata + m->outer_l2_len +
 190                                                        m->outer_l3_len + 4);
 191                                 *oudplen = rte_cpu_to_be_16(
 192                                         rte_be_to_cpu_16(*oudplen) - paylen);
 193                         }
 194
 195                         /* Update iplen position to inner ip hdr */
 196                         iplen = (uint16_t *)(mdata + lso_sb - m->l3_len -
 197                                              m->l4_len +
 198                                              (2 << !!(ol_flags & PKT_TX_IPV6)));
 199                 }
 200
 201                 *iplen = rte_cpu_to_be_16(rte_be_to_cpu_16(*iplen) - paylen);
 202         }
 203 }
 204
 205 static __rte_always_inline void
 206 cn10k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, uintptr_t lmt_addr,
 207                        const uint16_t flags, const uint64_t lso_tun_fmt)
 208 {
 209         struct nix_send_ext_s *send_hdr_ext;
 210         struct nix_send_hdr_s *send_hdr;
 211         uint64_t ol_flags = 0, mask;
 212         union nix_send_hdr_w1_u w1;
 213         union nix_send_sg_s *sg;
 214
 215         send_hdr = (struct nix_send_hdr_s *)cmd;
 216         if (flags & NIX_TX_NEED_EXT_HDR) {
 217                 send_hdr_ext = (struct nix_send_ext_s *)(cmd + 2);
 218                 sg = (union nix_send_sg_s *)(cmd + 4);
 219                 /* Clear previous markings */
 220                 send_hdr_ext->w0.lso = 0;
 221                 send_hdr_ext->w1.u = 0;
 222         } else {
 223                 sg = (union nix_send_sg_s *)(cmd + 2);
 224         }
 225
 226         if (flags & NIX_TX_NEED_SEND_HDR_W1) {
 227                 ol_flags = m->ol_flags;
 228                 w1.u = 0;
 229         }
 230
 231         if (!(flags & NIX_TX_MULTI_SEG_F)) {
 232                 send_hdr->w0.total = m->data_len;
 233                 send_hdr->w0.aura =
 234                         roc_npa_aura_handle_to_aura(m->pool->pool_id);
 235         }
 236
 237         /*
 238          * L3type:  2 => IPV4
 239          *          3 => IPV4 with csum
 240          *          4 => IPV6
 241          * L3type and L3ptr needs to be set for either
 242          * L3 csum or L4 csum or LSO
 243          *
 244          */
 245
 246         if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
 247             (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) {
 248                 const uint8_t csum = !!(ol_flags & PKT_TX_OUTER_UDP_CKSUM);
 249                 const uint8_t ol3type =
 250                         ((!!(ol_flags & PKT_TX_OUTER_IPV4)) << 1) +
 251                         ((!!(ol_flags & PKT_TX_OUTER_IPV6)) << 2) +
 252                         !!(ol_flags & PKT_TX_OUTER_IP_CKSUM);
 253
 254                 /* Outer L3 */
 255                 w1.ol3type = ol3type;
 256                 mask = 0xffffull << ((!!ol3type) << 4);
 257                 w1.ol3ptr = ~mask & m->outer_l2_len;
 258                 w1.ol4ptr = ~mask & (w1.ol3ptr + m->outer_l3_len);
 259
 260                 /* Outer L4 */
 261                 w1.ol4type = csum + (csum << 1);
 262
 263                 /* Inner L3 */
 264                 w1.il3type = ((!!(ol_flags & PKT_TX_IPV4)) << 1) +
 265                              ((!!(ol_flags & PKT_TX_IPV6)) << 2);
 266                 w1.il3ptr = w1.ol4ptr + m->l2_len;
 267                 w1.il4ptr = w1.il3ptr + m->l3_len;
 268                 /* Increment it by 1 if it is IPV4 as 3 is with csum */
 269                 w1.il3type = w1.il3type + !!(ol_flags & PKT_TX_IP_CKSUM);
 270
 271                 /* Inner L4 */
 272                 w1.il4type = (ol_flags & PKT_TX_L4_MASK) >> 52;
 273
 274                 /* In case of no tunnel header use only
 275                  * shift IL3/IL4 fields a bit to use
 276                  * OL3/OL4 for header checksum
 277                  */
 278                 mask = !ol3type;
 279                 w1.u = ((w1.u & 0xFFFFFFFF00000000) >> (mask << 3)) |
 280                        ((w1.u & 0X00000000FFFFFFFF) >> (mask << 4));
 281
 282         } else if (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) {
 283                 const uint8_t csum = !!(ol_flags & PKT_TX_OUTER_UDP_CKSUM);
 284                 const uint8_t outer_l2_len = m->outer_l2_len;
 285
 286                 /* Outer L3 */
 287                 w1.ol3ptr = outer_l2_len;
 288                 w1.ol4ptr = outer_l2_len + m->outer_l3_len;
 289                 /* Increment it by 1 if it is IPV4 as 3 is with csum */
 290                 w1.ol3type = ((!!(ol_flags & PKT_TX_OUTER_IPV4)) << 1) +
 291                              ((!!(ol_flags & PKT_TX_OUTER_IPV6)) << 2) +
 292                              !!(ol_flags & PKT_TX_OUTER_IP_CKSUM);
 293
 294                 /* Outer L4 */
 295                 w1.ol4type = csum + (csum << 1);
 296
 297         } else if (flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) {
 298                 const uint8_t l2_len = m->l2_len;
 299
 300                 /* Always use OLXPTR and OLXTYPE when only
 301                  * when one header is present
 302                  */
 303
 304                 /* Inner L3 */
 305                 w1.ol3ptr = l2_len;
 306                 w1.ol4ptr = l2_len + m->l3_len;
 307                 /* Increment it by 1 if it is IPV4 as 3 is with csum */
 308                 w1.ol3type = ((!!(ol_flags & PKT_TX_IPV4)) << 1) +
 309                              ((!!(ol_flags & PKT_TX_IPV6)) << 2) +
 310                              !!(ol_flags & PKT_TX_IP_CKSUM);
 311
 312                 /* Inner L4 */
 313                 w1.ol4type = (ol_flags & PKT_TX_L4_MASK) >> 52;
 314         }
 315
 316         if (flags & NIX_TX_NEED_EXT_HDR && flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
 317                 send_hdr_ext->w1.vlan1_ins_ena = !!(ol_flags & PKT_TX_VLAN);
 318                 /* HW will update ptr after vlan0 update */
 319                 send_hdr_ext->w1.vlan1_ins_ptr = 12;
 320                 send_hdr_ext->w1.vlan1_ins_tci = m->vlan_tci;
 321
 322                 send_hdr_ext->w1.vlan0_ins_ena = !!(ol_flags & PKT_TX_QINQ);
 323                 /* 2B before end of l2 header */
 324                 send_hdr_ext->w1.vlan0_ins_ptr = 12;
 325                 send_hdr_ext->w1.vlan0_ins_tci = m->vlan_tci_outer;
 326         }
 327
 328         if (flags & NIX_TX_OFFLOAD_TSO_F && (ol_flags & PKT_TX_TCP_SEG)) {
 329                 uint16_t lso_sb;
 330                 uint64_t mask;
 331
 332                 mask = -(!w1.il3type);
 333                 lso_sb = (mask & w1.ol4ptr) + (~mask & w1.il4ptr) + m->l4_len;
 334
 335                 send_hdr_ext->w0.lso_sb = lso_sb;
 336                 send_hdr_ext->w0.lso = 1;
 337                 send_hdr_ext->w0.lso_mps = m->tso_segsz;
 338                 send_hdr_ext->w0.lso_format =
 339                         NIX_LSO_FORMAT_IDX_TSOV4 + !!(ol_flags & PKT_TX_IPV6);
 340                 w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
 341
 342                 /* Handle tunnel tso */
 343                 if ((flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F) &&
 344                     (ol_flags & PKT_TX_TUNNEL_MASK)) {
 345                         const uint8_t is_udp_tun =
 346                                 (CNXK_NIX_UDP_TUN_BITMASK >>
 347                                  ((ol_flags & PKT_TX_TUNNEL_MASK) >> 45)) &
 348                                 0x1;
 349                         uint8_t shift = is_udp_tun ? 32 : 0;
 350
 351                         shift += (!!(ol_flags & PKT_TX_OUTER_IPV6) << 4);
 352                         shift += (!!(ol_flags & PKT_TX_IPV6) << 3);
 353
 354                         w1.il4type = NIX_SENDL4TYPE_TCP_CKSUM;
 355                         w1.ol4type = is_udp_tun ? NIX_SENDL4TYPE_UDP_CKSUM : 0;
 356                         /* Update format for UDP tunneled packet */
 357                         send_hdr_ext->w0.lso_format = (lso_tun_fmt >> shift);
 358                 }
 359         }
 360
 361         if (flags & NIX_TX_NEED_SEND_HDR_W1)
 362                 send_hdr->w1.u = w1.u;
 363
 364         if (!(flags & NIX_TX_MULTI_SEG_F)) {
 365                 sg->seg1_size = m->data_len;
 366                 *(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m);
 367
 368                 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
 369                         /* DF bit = 1 if refcount of current mbuf or parent mbuf
 370                          *              is greater than 1
 371                          * DF bit = 0 otherwise
 372                          */
 373                         send_hdr->w0.df = cnxk_nix_prefree_seg(m);
 374                 }
 375                 /* Mark mempool object as "put" since it is freed by NIX */
 376                 if (!send_hdr->w0.df)
 377                         __mempool_check_cookies(m->pool, (void **)&m, 1, 0);
 378         }
 379
 380         /* With minimal offloads, 'cmd' being local could be optimized out to
 381          * registers. In other cases, 'cmd' will be in stack. Intent is
 382          * 'cmd' stores content from txq->cmd which is copied only once.
 383          */
 384         *((struct nix_send_hdr_s *)lmt_addr) = *send_hdr;
 385         lmt_addr += 16;
 386         if (flags & NIX_TX_NEED_EXT_HDR) {
 387                 *((struct nix_send_ext_s *)lmt_addr) = *send_hdr_ext;
 388                 lmt_addr += 16;
 389         }
 390         /* In case of multi-seg, sg template is stored here */
 391         *((union nix_send_sg_s *)lmt_addr) = *sg;
 392         *(rte_iova_t *)(lmt_addr + 8) = *(rte_iova_t *)(sg + 1);
 393 }
 394
 395 static __rte_always_inline void
 396 cn10k_nix_xmit_prepare_tstamp(uintptr_t lmt_addr, const uint64_t *cmd,
 397                               const uint64_t ol_flags, const uint16_t no_segdw,
 398                               const uint16_t flags)
 399 {
 400         if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
 401                 const uint8_t is_ol_tstamp = !(ol_flags & PKT_TX_IEEE1588_TMST);
 402                 struct nix_send_ext_s *send_hdr_ext =
 403                                         (struct nix_send_ext_s *)lmt_addr + 16;
 404                 uint64_t *lmt = (uint64_t *)lmt_addr;
 405                 uint16_t off = (no_segdw - 1) << 1;
 406                 struct nix_send_mem_s *send_mem;
 407
 408                 send_mem = (struct nix_send_mem_s *)(lmt + off);
 409                 send_hdr_ext->w0.subdc = NIX_SUBDC_EXT;
 410                 send_hdr_ext->w0.tstmp = 1;
 411                 if (flags & NIX_TX_MULTI_SEG_F) {
 412                         /* Retrieving the default desc values */
 413                         lmt[off] = cmd[2];
 414
 415                         /* Using compiler barier to avoid voilation of C
 416                          * aliasing rules.
 417                          */
 418                         rte_compiler_barrier();
 419                 }
 420
 421                 /* Packets for which PKT_TX_IEEE1588_TMST is not set, tx tstamp
 422                  * should not be recorded, hence changing the alg type to
 423                  * NIX_SENDMEMALG_SET and also changing send mem addr field to
 424                  * next 8 bytes as it corrpt the actual tx tstamp registered
 425                  * address.
 426                  */
 427                 send_mem->w0.subdc = NIX_SUBDC_MEM;
 428                 send_mem->w0.alg = NIX_SENDMEMALG_SETTSTMP - (is_ol_tstamp);
 429                 send_mem->addr =
 430                         (rte_iova_t)(((uint64_t *)cmd[3]) + is_ol_tstamp);
 431         }
 432 }
 433
 434 static __rte_always_inline uint16_t
 435 cn10k_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
 436 {
 437         struct nix_send_hdr_s *send_hdr;
 438         union nix_send_sg_s *sg;
 439         struct rte_mbuf *m_next;
 440         uint64_t *slist, sg_u;
 441         uint64_t nb_segs;
 442         uint64_t segdw;
 443         uint8_t off, i;
 444
 445         send_hdr = (struct nix_send_hdr_s *)cmd;
 446         send_hdr->w0.total = m->pkt_len;
 447         send_hdr->w0.aura = roc_npa_aura_handle_to_aura(m->pool->pool_id);
 448
 449         if (flags & NIX_TX_NEED_EXT_HDR)
 450                 off = 2;
 451         else
 452                 off = 0;
 453
 454         sg = (union nix_send_sg_s *)&cmd[2 + off];
 455         /* Clear sg->u header before use */
 456         sg->u &= 0xFC00000000000000;
 457         sg_u = sg->u;
 458         slist = &cmd[3 + off];
 459
 460         i = 0;
 461         nb_segs = m->nb_segs;
 462
 463         /* Fill mbuf segments */
 464         do {
 465                 m_next = m->next;
 466                 sg_u = sg_u | ((uint64_t)m->data_len << (i << 4));
 467                 *slist = rte_mbuf_data_iova(m);
 468                 /* Set invert df if buffer is not to be freed by H/W */
 469                 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)
 470                         sg_u |= (cnxk_nix_prefree_seg(m) << (i + 55));
 471                         /* Mark mempool object as "put" since it is freed by NIX
 472                          */
 473 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
 474                 if (!(sg_u & (1ULL << (i + 55))))
 475                         __mempool_check_cookies(m->pool, (void **)&m, 1, 0);
 476 #endif
 477                 slist++;
 478                 i++;
 479                 nb_segs--;
 480                 if (i > 2 && nb_segs) {
 481                         i = 0;
 482                         /* Next SG subdesc */
 483                         *(uint64_t *)slist = sg_u & 0xFC00000000000000;
 484                         sg->u = sg_u;
 485                         sg->segs = 3;
 486                         sg = (union nix_send_sg_s *)slist;
 487                         sg_u = sg->u;
 488                         slist++;
 489                 }
 490                 m = m_next;
 491         } while (nb_segs);
 492
 493         sg->u = sg_u;
 494         sg->segs = i;
 495         segdw = (uint64_t *)slist - (uint64_t *)&cmd[2 + off];
 496         /* Roundup extra dwords to multiple of 2 */
 497         segdw = (segdw >> 1) + (segdw & 0x1);
 498         /* Default dwords */
 499         segdw += (off >> 1) + 1 + !!(flags & NIX_TX_OFFLOAD_TSTAMP_F);
 500         send_hdr->w0.sizem1 = segdw - 1;
 501
 502         return segdw;
 503 }
 504
 505 static __rte_always_inline uint16_t
 506 cn10k_nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts,
 507                     uint64_t *cmd, const uint16_t flags)
 508 {
 509         struct cn10k_eth_txq *txq = tx_queue;
 510         const rte_iova_t io_addr = txq->io_addr;
 511         uintptr_t pa, lmt_addr = txq->lmt_base;
 512         uint16_t lmt_id, burst, left, i;
 513         uint64_t lso_tun_fmt;
 514         uint64_t data;
 515
 516         NIX_XMIT_FC_OR_RETURN(txq, pkts);
 517
 518         /* Get cmd skeleton */
 519         cn10k_nix_tx_skeleton(txq, cmd, flags);
 520
 521         /* Reduce the cached count */
 522         txq->fc_cache_pkts -= pkts;
 523
 524         if (flags & NIX_TX_OFFLOAD_TSO_F)
 525                 lso_tun_fmt = txq->lso_tun_fmt;
 526
 527         /* Get LMT base address and LMT ID as lcore id */
 528         ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
 529         left = pkts;
 530 again:
 531         burst = left > 32 ? 32 : left;
 532         for (i = 0; i < burst; i++) {
 533                 /* Perform header writes for TSO, barrier at
 534                  * lmt steorl will suffice.
 535                  */
 536                 if (flags & NIX_TX_OFFLOAD_TSO_F)
 537                         cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags);
 538
 539                 cn10k_nix_xmit_prepare(tx_pkts[i], cmd, lmt_addr, flags,
 540                                        lso_tun_fmt);
 541                 cn10k_nix_xmit_prepare_tstamp(lmt_addr, &txq->cmd[0],
 542                                               tx_pkts[i]->ol_flags, 4, flags);
 543                 lmt_addr += (1ULL << ROC_LMT_LINE_SIZE_LOG2);
 544         }
 545
 546         /* Trigger LMTST */
 547         if (burst > 16) {
 548                 data = cn10k_nix_tx_steor_data(flags);
 549                 pa = io_addr | (data & 0x7) << 4;
 550                 data &= ~0x7ULL;
 551                 data |= (15ULL << 12);
 552                 data |= (uint64_t)lmt_id;
 553
 554                 /* STEOR0 */
 555                 roc_lmt_submit_steorl(data, pa);
 556
 557                 data = cn10k_nix_tx_steor_data(flags);
 558                 pa = io_addr | (data & 0x7) << 4;
 559                 data &= ~0x7ULL;
 560                 data |= ((uint64_t)(burst - 17)) << 12;
 561                 data |= (uint64_t)(lmt_id + 16);
 562
 563                 /* STEOR1 */
 564                 roc_lmt_submit_steorl(data, pa);
 565         } else if (burst) {
 566                 data = cn10k_nix_tx_steor_data(flags);
 567                 pa = io_addr | (data & 0x7) << 4;
 568                 data &= ~0x7ULL;
 569                 data |= ((uint64_t)(burst - 1)) << 12;
 570                 data |= lmt_id;
 571
 572                 /* STEOR0 */
 573                 roc_lmt_submit_steorl(data, pa);
 574         }
 575
 576         left -= burst;
 577         rte_io_wmb();
 578         if (left) {
 579                 /* Start processing another burst */
 580                 tx_pkts += burst;
 581                 /* Reset lmt base addr */
 582                 lmt_addr -= (1ULL << ROC_LMT_LINE_SIZE_LOG2);
 583                 lmt_addr &= (~(BIT_ULL(ROC_LMT_BASE_PER_CORE_LOG2) - 1));
 584                 goto again;
 585         }
 586
 587         return pkts;
 588 }
 589
 590 static __rte_always_inline uint16_t
 591 cn10k_nix_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts,
 592                          uint16_t pkts, uint64_t *cmd, const uint16_t flags)
 593 {
 594         struct cn10k_eth_txq *txq = tx_queue;
 595         uintptr_t pa0, pa1, lmt_addr = txq->lmt_base;
 596         const rte_iova_t io_addr = txq->io_addr;
 597         uint16_t segdw, lmt_id, burst, left, i;
 598         uint64_t data0, data1;
 599         uint64_t lso_tun_fmt;
 600         __uint128_t data128;
 601         uint16_t shft;
 602
 603         NIX_XMIT_FC_OR_RETURN(txq, pkts);
 604
 605         cn10k_nix_tx_skeleton(txq, cmd, flags);
 606
 607         /* Reduce the cached count */
 608         txq->fc_cache_pkts -= pkts;
 609
 610         if (flags & NIX_TX_OFFLOAD_TSO_F)
 611                 lso_tun_fmt = txq->lso_tun_fmt;
 612
 613         /* Get LMT base address and LMT ID as lcore id */
 614         ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
 615         left = pkts;
 616 again:
 617         burst = left > 32 ? 32 : left;
 618         shft = 16;
 619         data128 = 0;
 620         for (i = 0; i < burst; i++) {
 621                 /* Perform header writes for TSO, barrier at
 622                  * lmt steorl will suffice.
 623                  */
 624                 if (flags & NIX_TX_OFFLOAD_TSO_F)
 625                         cn10k_nix_xmit_prepare_tso(tx_pkts[i], flags);
 626
 627                 cn10k_nix_xmit_prepare(tx_pkts[i], cmd, lmt_addr, flags,
 628                                        lso_tun_fmt);
 629                 /* Store sg list directly on lmt line */
 630                 segdw = cn10k_nix_prepare_mseg(tx_pkts[i], (uint64_t *)lmt_addr,
 631                                                flags);
 632                 cn10k_nix_xmit_prepare_tstamp(lmt_addr, &txq->cmd[0],
 633                                               tx_pkts[i]->ol_flags, segdw,
 634                                               flags);
 635                 lmt_addr += (1ULL << ROC_LMT_LINE_SIZE_LOG2);
 636                 data128 |= (((__uint128_t)(segdw - 1)) << shft);
 637                 shft += 3;
 638         }
 639
 640         data0 = (uint64_t)data128;
 641         data1 = (uint64_t)(data128 >> 64);
 642         /* Make data0 similar to data1 */
 643         data0 >>= 16;
 644         /* Trigger LMTST */
 645         if (burst > 16) {
 646                 pa0 = io_addr | (data0 & 0x7) << 4;
 647                 data0 &= ~0x7ULL;
 648                 /* Move lmtst1..15 sz to bits 63:19 */
 649                 data0 <<= 16;
 650                 data0 |= (15ULL << 12);
 651                 data0 |= (uint64_t)lmt_id;
 652
 653                 /* STEOR0 */
 654                 roc_lmt_submit_steorl(data0, pa0);
 655
 656                 pa1 = io_addr | (data1 & 0x7) << 4;
 657                 data1 &= ~0x7ULL;
 658                 data1 <<= 16;
 659                 data1 |= ((uint64_t)(burst - 17)) << 12;
 660                 data1 |= (uint64_t)(lmt_id + 16);
 661
 662                 /* STEOR1 */
 663                 roc_lmt_submit_steorl(data1, pa1);
 664         } else if (burst) {
 665                 pa0 = io_addr | (data0 & 0x7) << 4;
 666                 data0 &= ~0x7ULL;
 667                 /* Move lmtst1..15 sz to bits 63:19 */
 668                 data0 <<= 16;
 669                 data0 |= ((burst - 1) << 12);
 670                 data0 |= (uint64_t)lmt_id;
 671
 672                 /* STEOR0 */
 673                 roc_lmt_submit_steorl(data0, pa0);
 674         }
 675
 676         left -= burst;
 677         rte_io_wmb();
 678         if (left) {
 679                 /* Start processing another burst */
 680                 tx_pkts += burst;
 681                 /* Reset lmt base addr */
 682                 lmt_addr -= (1ULL << ROC_LMT_LINE_SIZE_LOG2);
 683                 lmt_addr &= (~(BIT_ULL(ROC_LMT_BASE_PER_CORE_LOG2) - 1));
 684                 goto again;
 685         }
 686
 687         return pkts;
 688 }
 689
 690 #if defined(RTE_ARCH_ARM64)
 691
 692 #define NIX_DESCS_PER_LOOP 4
 693 static __rte_always_inline uint16_t
 694 cn10k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
 695                            uint16_t pkts, uint64_t *cmd, const uint16_t flags)
 696 {
 697         uint64x2_t dataoff_iova0, dataoff_iova1, dataoff_iova2, dataoff_iova3;
 698         uint64x2_t len_olflags0, len_olflags1, len_olflags2, len_olflags3;
 699         uint64x2_t cmd0[NIX_DESCS_PER_LOOP], cmd1[NIX_DESCS_PER_LOOP],
 700                 cmd2[NIX_DESCS_PER_LOOP], cmd3[NIX_DESCS_PER_LOOP];
 701         uint64_t *mbuf0, *mbuf1, *mbuf2, *mbuf3, data, pa;
 702         uint64x2_t senddesc01_w0, senddesc23_w0;
 703         uint64x2_t senddesc01_w1, senddesc23_w1;
 704         uint16_t left, scalar, burst, i, lmt_id;
 705         uint64x2_t sendext01_w0, sendext23_w0;
 706         uint64x2_t sendext01_w1, sendext23_w1;
 707         uint64x2_t sendmem01_w0, sendmem23_w0;
 708         uint64x2_t sendmem01_w1, sendmem23_w1;
 709         uint64x2_t sgdesc01_w0, sgdesc23_w0;
 710         uint64x2_t sgdesc01_w1, sgdesc23_w1;
 711         struct cn10k_eth_txq *txq = tx_queue;
 712         uintptr_t laddr = txq->lmt_base;
 713         rte_iova_t io_addr = txq->io_addr;
 714         uint64x2_t ltypes01, ltypes23;
 715         uint64x2_t xtmp128, ytmp128;
 716         uint64x2_t xmask01, xmask23;
 717         uint8_t lnum;
 718
 719         NIX_XMIT_FC_OR_RETURN(txq, pkts);
 720
 721         scalar = pkts & (NIX_DESCS_PER_LOOP - 1);
 722         pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
 723
 724         /* Reduce the cached count */
 725         txq->fc_cache_pkts -= pkts;
 726
 727         senddesc01_w0 = vld1q_dup_u64(&txq->send_hdr_w0);
 728         senddesc23_w0 = senddesc01_w0;
 729         senddesc01_w1 = vdupq_n_u64(0);
 730         senddesc23_w1 = senddesc01_w1;
 731         sgdesc01_w0 = vld1q_dup_u64(&txq->sg_w0);
 732         sgdesc23_w0 = sgdesc01_w0;
 733
 734         /* Load command defaults into vector variables. */
 735         if (flags & NIX_TX_NEED_EXT_HDR) {
 736                 sendext01_w0 = vld1q_dup_u64(&txq->cmd[0]);
 737                 sendext23_w0 = sendext01_w0;
 738                 sendext01_w1 = vdupq_n_u64(12 | 12U << 24);
 739                 sendext23_w1 = sendext01_w1;
 740                 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
 741                         sendmem01_w0 = vld1q_dup_u64(&txq->cmd[2]);
 742                         sendmem23_w0 = sendmem01_w0;
 743                         sendmem01_w1 = vld1q_dup_u64(&txq->cmd[3]);
 744                         sendmem23_w1 = sendmem01_w1;
 745                 }
 746         }
 747
 748         /* Get LMT base address and LMT ID as lcore id */
 749         ROC_LMT_BASE_ID_GET(laddr, lmt_id);
 750         left = pkts;
 751 again:
 752         /* Number of packets to prepare depends on offloads enabled. */
 753         burst = left > cn10k_nix_pkts_per_vec_brst(flags) ?
 754                               cn10k_nix_pkts_per_vec_brst(flags) :
 755                               left;
 756         lnum = 0;
 757         for (i = 0; i < burst; i += NIX_DESCS_PER_LOOP) {
 758                 /* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */
 759                 senddesc01_w0 =
 760                         vbicq_u64(senddesc01_w0, vdupq_n_u64(0xFFFFFFFF));
 761                 sgdesc01_w0 = vbicq_u64(sgdesc01_w0, vdupq_n_u64(0xFFFFFFFF));
 762
 763                 senddesc23_w0 = senddesc01_w0;
 764                 sgdesc23_w0 = sgdesc01_w0;
 765
 766                 /* Clear vlan enables. */
 767                 if (flags & NIX_TX_NEED_EXT_HDR) {
 768                         sendext01_w1 = vbicq_u64(sendext01_w1,
 769                                                  vdupq_n_u64(0x3FFFF00FFFF00));
 770                         sendext23_w1 = sendext01_w1;
 771                 }
 772
 773                 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
 774                         /* Reset send mem alg to SETTSTMP from SUB*/
 775                         sendmem01_w0 = vbicq_u64(sendmem01_w0,
 776                                                  vdupq_n_u64(BIT_ULL(59)));
 777                         /* Reset send mem address to default. */
 778                         sendmem01_w1 =
 779                                 vbicq_u64(sendmem01_w1, vdupq_n_u64(0xF));
 780                         sendmem23_w0 = sendmem01_w0;
 781                         sendmem23_w1 = sendmem01_w1;
 782                 }
 783
 784                 /* Move mbufs to iova */
 785                 mbuf0 = (uint64_t *)tx_pkts[0];
 786                 mbuf1 = (uint64_t *)tx_pkts[1];
 787                 mbuf2 = (uint64_t *)tx_pkts[2];
 788                 mbuf3 = (uint64_t *)tx_pkts[3];
 789
 790                 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
 791                                      offsetof(struct rte_mbuf, buf_iova));
 792                 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
 793                                      offsetof(struct rte_mbuf, buf_iova));
 794                 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
 795                                      offsetof(struct rte_mbuf, buf_iova));
 796                 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
 797                                      offsetof(struct rte_mbuf, buf_iova));
 798                 /*
 799                  * Get mbuf's, olflags, iova, pktlen, dataoff
 800                  * dataoff_iovaX.D[0] = iova,
 801                  * dataoff_iovaX.D[1](15:0) = mbuf->dataoff
 802                  * len_olflagsX.D[0] = ol_flags,
 803                  * len_olflagsX.D[1](63:32) = mbuf->pkt_len
 804                  */
 805                 dataoff_iova0 = vld1q_u64(mbuf0);
 806                 len_olflags0 = vld1q_u64(mbuf0 + 2);
 807                 dataoff_iova1 = vld1q_u64(mbuf1);
 808                 len_olflags1 = vld1q_u64(mbuf1 + 2);
 809                 dataoff_iova2 = vld1q_u64(mbuf2);
 810                 len_olflags2 = vld1q_u64(mbuf2 + 2);
 811                 dataoff_iova3 = vld1q_u64(mbuf3);
 812                 len_olflags3 = vld1q_u64(mbuf3 + 2);
 813
 814                 /* Move mbufs to point pool */
 815                 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
 816                                      offsetof(struct rte_mbuf, pool) -
 817                                      offsetof(struct rte_mbuf, buf_iova));
 818                 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
 819                                      offsetof(struct rte_mbuf, pool) -
 820                                      offsetof(struct rte_mbuf, buf_iova));
 821                 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
 822                                      offsetof(struct rte_mbuf, pool) -
 823                                      offsetof(struct rte_mbuf, buf_iova));
 824                 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
 825                                      offsetof(struct rte_mbuf, pool) -
 826                                      offsetof(struct rte_mbuf, buf_iova));
 827
 828                 if (flags & (NIX_TX_OFFLOAD_OL3_OL4_CSUM_F |
 829                              NIX_TX_OFFLOAD_L3_L4_CSUM_F)) {
 830                         /* Get tx_offload for ol2, ol3, l2, l3 lengths */
 831                         /*
 832                          * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7)
 833                          * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7)
 834                          */
 835
 836                         asm volatile("LD1 {%[a].D}[0],[%[in]]\n\t"
 837                                      : [a] "+w"(senddesc01_w1)
 838                                      : [in] "r"(mbuf0 + 2)
 839                                      : "memory");
 840
 841                         asm volatile("LD1 {%[a].D}[1],[%[in]]\n\t"
 842                                      : [a] "+w"(senddesc01_w1)
 843                                      : [in] "r"(mbuf1 + 2)
 844                                      : "memory");
 845
 846                         asm volatile("LD1 {%[b].D}[0],[%[in]]\n\t"
 847                                      : [b] "+w"(senddesc23_w1)
 848                                      : [in] "r"(mbuf2 + 2)
 849                                      : "memory");
 850
 851                         asm volatile("LD1 {%[b].D}[1],[%[in]]\n\t"
 852                                      : [b] "+w"(senddesc23_w1)
 853                                      : [in] "r"(mbuf3 + 2)
 854                                      : "memory");
 855
 856                         /* Get pool pointer alone */
 857                         mbuf0 = (uint64_t *)*mbuf0;
 858                         mbuf1 = (uint64_t *)*mbuf1;
 859                         mbuf2 = (uint64_t *)*mbuf2;
 860                         mbuf3 = (uint64_t *)*mbuf3;
 861                 } else {
 862                         /* Get pool pointer alone */
 863                         mbuf0 = (uint64_t *)*mbuf0;
 864                         mbuf1 = (uint64_t *)*mbuf1;
 865                         mbuf2 = (uint64_t *)*mbuf2;
 866                         mbuf3 = (uint64_t *)*mbuf3;
 867                 }
 868
 869                 const uint8x16_t shuf_mask2 = {
 870                         0x4, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 871                         0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 872                 };
 873                 xtmp128 = vzip2q_u64(len_olflags0, len_olflags1);
 874                 ytmp128 = vzip2q_u64(len_olflags2, len_olflags3);
 875
 876                 /* Clear dataoff_iovaX.D[1] bits other than dataoff(15:0) */
 877                 const uint64x2_t and_mask0 = {
 878                         0xFFFFFFFFFFFFFFFF,
 879                         0x000000000000FFFF,
 880                 };
 881
 882                 dataoff_iova0 = vandq_u64(dataoff_iova0, and_mask0);
 883                 dataoff_iova1 = vandq_u64(dataoff_iova1, and_mask0);
 884                 dataoff_iova2 = vandq_u64(dataoff_iova2, and_mask0);
 885                 dataoff_iova3 = vandq_u64(dataoff_iova3, and_mask0);
 886
 887                 /*
 888                  * Pick only 16 bits of pktlen preset at bits 63:32
 889                  * and place them at bits 15:0.
 890                  */
 891                 xtmp128 = vqtbl1q_u8(xtmp128, shuf_mask2);
 892                 ytmp128 = vqtbl1q_u8(ytmp128, shuf_mask2);
 893
 894                 /* Add pairwise to get dataoff + iova in sgdesc_w1 */
 895                 sgdesc01_w1 = vpaddq_u64(dataoff_iova0, dataoff_iova1);
 896                 sgdesc23_w1 = vpaddq_u64(dataoff_iova2, dataoff_iova3);
 897
 898                 /* Orr both sgdesc_w0 and senddesc_w0 with 16 bits of
 899                  * pktlen at 15:0 position.
 900                  */
 901                 sgdesc01_w0 = vorrq_u64(sgdesc01_w0, xtmp128);
 902                 sgdesc23_w0 = vorrq_u64(sgdesc23_w0, ytmp128);
 903                 senddesc01_w0 = vorrq_u64(senddesc01_w0, xtmp128);
 904                 senddesc23_w0 = vorrq_u64(senddesc23_w0, ytmp128);
 905
 906                 /* Move mbuf to point to pool_id. */
 907                 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
 908                                      offsetof(struct rte_mempool, pool_id));
 909                 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
 910                                      offsetof(struct rte_mempool, pool_id));
 911                 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
 912                                      offsetof(struct rte_mempool, pool_id));
 913                 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
 914                                      offsetof(struct rte_mempool, pool_id));
 915
 916                 if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
 917                     !(flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
 918                         /*
 919                          * Lookup table to translate ol_flags to
 920                          * il3/il4 types. But we still use ol3/ol4 types in
 921                          * senddesc_w1 as only one header processing is enabled.
 922                          */
 923                         const uint8x16_t tbl = {
 924                                 /* [0-15] = il4type:il3type */
 925                                 0x04, /* none (IPv6 assumed) */
 926                                 0x14, /* PKT_TX_TCP_CKSUM (IPv6 assumed) */
 927                                 0x24, /* PKT_TX_SCTP_CKSUM (IPv6 assumed) */
 928                                 0x34, /* PKT_TX_UDP_CKSUM (IPv6 assumed) */
 929                                 0x03, /* PKT_TX_IP_CKSUM */
 930                                 0x13, /* PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM */
 931                                 0x23, /* PKT_TX_IP_CKSUM | PKT_TX_SCTP_CKSUM */
 932                                 0x33, /* PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM */
 933                                 0x02, /* PKT_TX_IPV4  */
 934                                 0x12, /* PKT_TX_IPV4 | PKT_TX_TCP_CKSUM */
 935                                 0x22, /* PKT_TX_IPV4 | PKT_TX_SCTP_CKSUM */
 936                                 0x32, /* PKT_TX_IPV4 | PKT_TX_UDP_CKSUM */
 937                                 0x03, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM */
 938                                 0x13, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
 939                                        * PKT_TX_TCP_CKSUM
 940                                        */
 941                                 0x23, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
 942                                        * PKT_TX_SCTP_CKSUM
 943                                        */
 944                                 0x33, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
 945                                        * PKT_TX_UDP_CKSUM
 946                                        */
 947                         };
 948
 949                         /* Extract olflags to translate to iltypes */
 950                         xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
 951                         ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
 952
 953                         /*
 954                          * E(47):L3_LEN(9):L2_LEN(7+z)
 955                          * E(47):L3_LEN(9):L2_LEN(7+z)
 956                          */
 957                         senddesc01_w1 = vshlq_n_u64(senddesc01_w1, 1);
 958                         senddesc23_w1 = vshlq_n_u64(senddesc23_w1, 1);
 959
 960                         /* Move OLFLAGS bits 55:52 to 51:48
 961                          * with zeros preprended on the byte and rest
 962                          * don't care
 963                          */
 964                         xtmp128 = vshrq_n_u8(xtmp128, 4);
 965                         ytmp128 = vshrq_n_u8(ytmp128, 4);
 966                         /*
 967                          * E(48):L3_LEN(8):L2_LEN(z+7)
 968                          * E(48):L3_LEN(8):L2_LEN(z+7)
 969                          */
 970                         const int8x16_t tshft3 = {
 971                                 -1, 0, 8, 8, 8, 8, 8, 8,
 972                                 -1, 0, 8, 8, 8, 8, 8, 8,
 973                         };
 974
 975                         senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
 976                         senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
 977
 978                         /* Do the lookup */
 979                         ltypes01 = vqtbl1q_u8(tbl, xtmp128);
 980                         ltypes23 = vqtbl1q_u8(tbl, ytmp128);
 981
 982                         /* Pick only relevant fields i.e Bit 48:55 of iltype
 983                          * and place it in ol3/ol4type of senddesc_w1
 984                          */
 985                         const uint8x16_t shuf_mask0 = {
 986                                 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0xFF, 0xFF, 0xFF,
 987                                 0xFF, 0xFF, 0xFF, 0xFF, 0xE, 0xFF, 0xFF, 0xFF,
 988                         };
 989
 990                         ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
 991                         ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
 992
 993                         /* Prepare ol4ptr, ol3ptr from ol3len, ol2len.
 994                          * a [E(32):E(16):OL3(8):OL2(8)]
 995                          * a = a + (a << 8)
 996                          * a [E(32):E(16):(OL3+OL2):OL2]
 997                          * => E(32):E(16)::OL4PTR(8):OL3PTR(8)
 998                          */
 999                         senddesc01_w1 = vaddq_u8(senddesc01_w1,
1000                                                  vshlq_n_u16(senddesc01_w1, 8));
1001                         senddesc23_w1 = vaddq_u8(senddesc23_w1,
1002                                                  vshlq_n_u16(senddesc23_w1, 8));
1003
1004                         /* Move ltypes to senddesc*_w1 */
1005                         senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
1006                         senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
1007                 } else if (!(flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
1008                            (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
1009                         /*
1010                          * Lookup table to translate ol_flags to
1011                          * ol3/ol4 types.
1012                          */
1013
1014                         const uint8x16_t tbl = {
1015                                 /* [0-15] = ol4type:ol3type */
1016                                 0x00, /* none */
1017                                 0x03, /* OUTER_IP_CKSUM */
1018                                 0x02, /* OUTER_IPV4 */
1019                                 0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */
1020                                 0x04, /* OUTER_IPV6 */
1021                                 0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */
1022                                 0x00, /* OUTER_IPV6 | OUTER_IPV4 */
1023                                 0x00, /* OUTER_IPV6 | OUTER_IPV4 |
1024                                        * OUTER_IP_CKSUM
1025                                        */
1026                                 0x00, /* OUTER_UDP_CKSUM */
1027                                 0x33, /* OUTER_UDP_CKSUM | OUTER_IP_CKSUM */
1028                                 0x32, /* OUTER_UDP_CKSUM | OUTER_IPV4 */
1029                                 0x33, /* OUTER_UDP_CKSUM | OUTER_IPV4 |
1030                                        * OUTER_IP_CKSUM
1031                                        */
1032                                 0x34, /* OUTER_UDP_CKSUM | OUTER_IPV6 */
1033                                 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1034                                        * OUTER_IP_CKSUM
1035                                        */
1036                                 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1037                                        * OUTER_IPV4
1038                                        */
1039                                 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1040                                        * OUTER_IPV4 | OUTER_IP_CKSUM
1041                                        */
1042                         };
1043
1044                         /* Extract olflags to translate to iltypes */
1045                         xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1046                         ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1047
1048                         /*
1049                          * E(47):OL3_LEN(9):OL2_LEN(7+z)
1050                          * E(47):OL3_LEN(9):OL2_LEN(7+z)
1051                          */
1052                         const uint8x16_t shuf_mask5 = {
1053                                 0x6, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1054                                 0xE, 0xD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1055                         };
1056                         senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5);
1057                         senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5);
1058
1059                         /* Extract outer ol flags only */
1060                         const uint64x2_t o_cksum_mask = {
1061                                 0x1C00020000000000,
1062                                 0x1C00020000000000,
1063                         };
1064
1065                         xtmp128 = vandq_u64(xtmp128, o_cksum_mask);
1066                         ytmp128 = vandq_u64(ytmp128, o_cksum_mask);
1067
1068                         /* Extract OUTER_UDP_CKSUM bit 41 and
1069                          * move it to bit 61
1070                          */
1071
1072                         xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20);
1073                         ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20);
1074
1075                         /* Shift oltype by 2 to start nibble from BIT(56)
1076                          * instead of BIT(58)
1077                          */
1078                         xtmp128 = vshrq_n_u8(xtmp128, 2);
1079                         ytmp128 = vshrq_n_u8(ytmp128, 2);
1080                         /*
1081                          * E(48):L3_LEN(8):L2_LEN(z+7)
1082                          * E(48):L3_LEN(8):L2_LEN(z+7)
1083                          */
1084                         const int8x16_t tshft3 = {
1085                                 -1, 0, 8, 8, 8, 8, 8, 8,
1086                                 -1, 0, 8, 8, 8, 8, 8, 8,
1087                         };
1088
1089                         senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
1090                         senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
1091
1092                         /* Do the lookup */
1093                         ltypes01 = vqtbl1q_u8(tbl, xtmp128);
1094                         ltypes23 = vqtbl1q_u8(tbl, ytmp128);
1095
1096                         /* Pick only relevant fields i.e Bit 56:63 of oltype
1097                          * and place it in ol3/ol4type of senddesc_w1
1098                          */
1099                         const uint8x16_t shuf_mask0 = {
1100                                 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0xFF, 0xFF, 0xFF,
1101                                 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xFF, 0xFF, 0xFF,
1102                         };
1103
1104                         ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
1105                         ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
1106
1107                         /* Prepare ol4ptr, ol3ptr from ol3len, ol2len.
1108                          * a [E(32):E(16):OL3(8):OL2(8)]
1109                          * a = a + (a << 8)
1110                          * a [E(32):E(16):(OL3+OL2):OL2]
1111                          * => E(32):E(16)::OL4PTR(8):OL3PTR(8)
1112                          */
1113                         senddesc01_w1 = vaddq_u8(senddesc01_w1,
1114                                                  vshlq_n_u16(senddesc01_w1, 8));
1115                         senddesc23_w1 = vaddq_u8(senddesc23_w1,
1116                                                  vshlq_n_u16(senddesc23_w1, 8));
1117
1118                         /* Move ltypes to senddesc*_w1 */
1119                         senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
1120                         senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
1121                 } else if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
1122                            (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
1123                         /* Lookup table to translate ol_flags to
1124                          * ol4type, ol3type, il4type, il3type of senddesc_w1
1125                          */
1126                         const uint8x16x2_t tbl = {{
1127                                 {
1128                                         /* [0-15] = il4type:il3type */
1129                                         0x04, /* none (IPv6) */
1130                                         0x14, /* PKT_TX_TCP_CKSUM (IPv6) */
1131                                         0x24, /* PKT_TX_SCTP_CKSUM (IPv6) */
1132                                         0x34, /* PKT_TX_UDP_CKSUM (IPv6) */
1133                                         0x03, /* PKT_TX_IP_CKSUM */
1134                                         0x13, /* PKT_TX_IP_CKSUM |
1135                                                * PKT_TX_TCP_CKSUM
1136                                                */
1137                                         0x23, /* PKT_TX_IP_CKSUM |
1138                                                * PKT_TX_SCTP_CKSUM
1139                                                */
1140                                         0x33, /* PKT_TX_IP_CKSUM |
1141                                                * PKT_TX_UDP_CKSUM
1142                                                */
1143                                         0x02, /* PKT_TX_IPV4 */
1144                                         0x12, /* PKT_TX_IPV4 |
1145                                                * PKT_TX_TCP_CKSUM
1146                                                */
1147                                         0x22, /* PKT_TX_IPV4 |
1148                                                * PKT_TX_SCTP_CKSUM
1149                                                */
1150                                         0x32, /* PKT_TX_IPV4 |
1151                                                * PKT_TX_UDP_CKSUM
1152                                                */
1153                                         0x03, /* PKT_TX_IPV4 |
1154                                                * PKT_TX_IP_CKSUM
1155                                                */
1156                                         0x13, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
1157                                                * PKT_TX_TCP_CKSUM
1158                                                */
1159                                         0x23, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
1160                                                * PKT_TX_SCTP_CKSUM
1161                                                */
1162                                         0x33, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
1163                                                * PKT_TX_UDP_CKSUM
1164                                                */
1165                                 },
1166
1167                                 {
1168                                         /* [16-31] = ol4type:ol3type */
1169                                         0x00, /* none */
1170                                         0x03, /* OUTER_IP_CKSUM */
1171                                         0x02, /* OUTER_IPV4 */
1172                                         0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */
1173                                         0x04, /* OUTER_IPV6 */
1174                                         0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */
1175                                         0x00, /* OUTER_IPV6 | OUTER_IPV4 */
1176                                         0x00, /* OUTER_IPV6 | OUTER_IPV4 |
1177                                                * OUTER_IP_CKSUM
1178                                                */
1179                                         0x00, /* OUTER_UDP_CKSUM */
1180                                         0x33, /* OUTER_UDP_CKSUM |
1181                                                * OUTER_IP_CKSUM
1182                                                */
1183                                         0x32, /* OUTER_UDP_CKSUM |
1184                                                * OUTER_IPV4
1185                                                */
1186                                         0x33, /* OUTER_UDP_CKSUM |
1187                                                * OUTER_IPV4 | OUTER_IP_CKSUM
1188                                                */
1189                                         0x34, /* OUTER_UDP_CKSUM |
1190                                                * OUTER_IPV6
1191                                                */
1192                                         0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1193                                                * OUTER_IP_CKSUM
1194                                                */
1195                                         0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1196                                                * OUTER_IPV4
1197                                                */
1198                                         0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
1199                                                * OUTER_IPV4 | OUTER_IP_CKSUM
1200                                                */
1201                                 },
1202                         }};
1203
1204                         /* Extract olflags to translate to oltype & iltype */
1205                         xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1206                         ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1207
1208                         /*
1209                          * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z)
1210                          * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z)
1211                          */
1212                         const uint32x4_t tshft_4 = {
1213                                 1,
1214                                 0,
1215                                 1,
1216                                 0,
1217                         };
1218                         senddesc01_w1 = vshlq_u32(senddesc01_w1, tshft_4);
1219                         senddesc23_w1 = vshlq_u32(senddesc23_w1, tshft_4);
1220
1221                         /*
1222                          * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z)
1223                          * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z)
1224                          */
1225                         const uint8x16_t shuf_mask5 = {
1226                                 0x6, 0x5, 0x0, 0x1, 0xFF, 0xFF, 0xFF, 0xFF,
1227                                 0xE, 0xD, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF,
1228                         };
1229                         senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5);
1230                         senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5);
1231
1232                         /* Extract outer and inner header ol_flags */
1233                         const uint64x2_t oi_cksum_mask = {
1234                                 0x1CF0020000000000,
1235                                 0x1CF0020000000000,
1236                         };
1237
1238                         xtmp128 = vandq_u64(xtmp128, oi_cksum_mask);
1239                         ytmp128 = vandq_u64(ytmp128, oi_cksum_mask);
1240
1241                         /* Extract OUTER_UDP_CKSUM bit 41 and
1242                          * move it to bit 61
1243                          */
1244
1245                         xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20);
1246                         ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20);
1247
1248                         /* Shift right oltype by 2 and iltype by 4
1249                          * to start oltype nibble from BIT(58)
1250                          * instead of BIT(56) and iltype nibble from BIT(48)
1251                          * instead of BIT(52).
1252                          */
1253                         const int8x16_t tshft5 = {
1254                                 8, 8, 8, 8, 8, 8, -4, -2,
1255                                 8, 8, 8, 8, 8, 8, -4, -2,
1256                         };
1257
1258                         xtmp128 = vshlq_u8(xtmp128, tshft5);
1259                         ytmp128 = vshlq_u8(ytmp128, tshft5);
1260                         /*
1261                          * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8)
1262                          * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8)
1263                          */
1264                         const int8x16_t tshft3 = {
1265                                 -1, 0, -1, 0, 0, 0, 0, 0,
1266                                 -1, 0, -1, 0, 0, 0, 0, 0,
1267                         };
1268
1269                         senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
1270                         senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
1271
1272                         /* Mark Bit(4) of oltype */
1273                         const uint64x2_t oi_cksum_mask2 = {
1274                                 0x1000000000000000,
1275                                 0x1000000000000000,
1276                         };
1277
1278                         xtmp128 = vorrq_u64(xtmp128, oi_cksum_mask2);
1279                         ytmp128 = vorrq_u64(ytmp128, oi_cksum_mask2);
1280
1281                         /* Do the lookup */
1282                         ltypes01 = vqtbl2q_u8(tbl, xtmp128);
1283                         ltypes23 = vqtbl2q_u8(tbl, ytmp128);
1284
1285                         /* Pick only relevant fields i.e Bit 48:55 of iltype and
1286                          * Bit 56:63 of oltype and place it in corresponding
1287                          * place in senddesc_w1.
1288                          */
1289                         const uint8x16_t shuf_mask0 = {
1290                                 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0x6, 0xFF, 0xFF,
1291                                 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xE, 0xFF, 0xFF,
1292                         };
1293
1294                         ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
1295                         ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
1296
1297                         /* Prepare l4ptr, l3ptr, ol4ptr, ol3ptr from
1298                          * l3len, l2len, ol3len, ol2len.
1299                          * a [E(32):L3(8):L2(8):OL3(8):OL2(8)]
1300                          * a = a + (a << 8)
1301                          * a [E:(L3+L2):(L2+OL3):(OL3+OL2):OL2]
1302                          * a = a + (a << 16)
1303                          * a [E:(L3+L2+OL3+OL2):(L2+OL3+OL2):(OL3+OL2):OL2]
1304                          * => E(32):IL4PTR(8):IL3PTR(8):OL4PTR(8):OL3PTR(8)
1305                          */
1306                         senddesc01_w1 = vaddq_u8(senddesc01_w1,
1307                                                  vshlq_n_u32(senddesc01_w1, 8));
1308                         senddesc23_w1 = vaddq_u8(senddesc23_w1,
1309                                                  vshlq_n_u32(senddesc23_w1, 8));
1310
1311                         /* Continue preparing l4ptr, l3ptr, ol4ptr, ol3ptr */
1312                         senddesc01_w1 = vaddq_u8(
1313                                 senddesc01_w1, vshlq_n_u32(senddesc01_w1, 16));
1314                         senddesc23_w1 = vaddq_u8(
1315                                 senddesc23_w1, vshlq_n_u32(senddesc23_w1, 16));
1316
1317                         /* Move ltypes to senddesc*_w1 */
1318                         senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
1319                         senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
1320                 }
1321
1322                 xmask01 = vdupq_n_u64(0);
1323                 xmask23 = xmask01;
1324                 asm volatile("LD1 {%[a].H}[0],[%[in]]\n\t"
1325                              : [a] "+w"(xmask01)
1326                              : [in] "r"(mbuf0)
1327                              : "memory");
1328
1329                 asm volatile("LD1 {%[a].H}[4],[%[in]]\n\t"
1330                              : [a] "+w"(xmask01)
1331                              : [in] "r"(mbuf1)
1332                              : "memory");
1333
1334                 asm volatile("LD1 {%[b].H}[0],[%[in]]\n\t"
1335                              : [b] "+w"(xmask23)
1336                              : [in] "r"(mbuf2)
1337                              : "memory");
1338
1339                 asm volatile("LD1 {%[b].H}[4],[%[in]]\n\t"
1340                              : [b] "+w"(xmask23)
1341                              : [in] "r"(mbuf3)
1342                              : "memory");
1343                 xmask01 = vshlq_n_u64(xmask01, 20);
1344                 xmask23 = vshlq_n_u64(xmask23, 20);
1345
1346                 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
1347                 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
1348
1349                 if (flags & NIX_TX_OFFLOAD_VLAN_QINQ_F) {
1350                         /* Tx ol_flag for vlan. */
1351                         const uint64x2_t olv = {PKT_TX_VLAN, PKT_TX_VLAN};
1352                         /* Bit enable for VLAN1 */
1353                         const uint64x2_t mlv = {BIT_ULL(49), BIT_ULL(49)};
1354                         /* Tx ol_flag for QnQ. */
1355                         const uint64x2_t olq = {PKT_TX_QINQ, PKT_TX_QINQ};
1356                         /* Bit enable for VLAN0 */
1357                         const uint64x2_t mlq = {BIT_ULL(48), BIT_ULL(48)};
1358                         /* Load vlan values from packet. outer is VLAN 0 */
1359                         uint64x2_t ext01 = {
1360                                 ((uint32_t)tx_pkts[0]->vlan_tci_outer) << 8 |
1361                                         ((uint64_t)tx_pkts[0]->vlan_tci) << 32,
1362                                 ((uint32_t)tx_pkts[1]->vlan_tci_outer) << 8 |
1363                                         ((uint64_t)tx_pkts[1]->vlan_tci) << 32,
1364                         };
1365                         uint64x2_t ext23 = {
1366                                 ((uint32_t)tx_pkts[2]->vlan_tci_outer) << 8 |
1367                                         ((uint64_t)tx_pkts[2]->vlan_tci) << 32,
1368                                 ((uint32_t)tx_pkts[3]->vlan_tci_outer) << 8 |
1369                                         ((uint64_t)tx_pkts[3]->vlan_tci) << 32,
1370                         };
1371
1372                         /* Get ol_flags of the packets. */
1373                         xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1374                         ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1375
1376                         /* ORR vlan outer/inner values into cmd. */
1377                         sendext01_w1 = vorrq_u64(sendext01_w1, ext01);
1378                         sendext23_w1 = vorrq_u64(sendext23_w1, ext23);
1379
1380                         /* Test for offload enable bits and generate masks. */
1381                         xtmp128 = vorrq_u64(vandq_u64(vtstq_u64(xtmp128, olv),
1382                                                       mlv),
1383                                             vandq_u64(vtstq_u64(xtmp128, olq),
1384                                                       mlq));
1385                         ytmp128 = vorrq_u64(vandq_u64(vtstq_u64(ytmp128, olv),
1386                                                       mlv),
1387                                             vandq_u64(vtstq_u64(ytmp128, olq),
1388                                                       mlq));
1389
1390                         /* Set vlan enable bits into cmd based on mask. */
1391                         sendext01_w1 = vorrq_u64(sendext01_w1, xtmp128);
1392                         sendext23_w1 = vorrq_u64(sendext23_w1, ytmp128);
1393                 }
1394
1395                 if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
1396                         /* Tx ol_flag for timestam. */
1397                         const uint64x2_t olf = {PKT_TX_IEEE1588_TMST,
1398                                                 PKT_TX_IEEE1588_TMST};
1399                         /* Set send mem alg to SUB. */
1400                         const uint64x2_t alg = {BIT_ULL(59), BIT_ULL(59)};
1401                         /* Increment send mem address by 8. */
1402                         const uint64x2_t addr = {0x8, 0x8};
1403
1404                         xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
1405                         ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
1406
1407                         /* Check if timestamp is requested and generate inverted
1408                          * mask as we need not make any changes to default cmd
1409                          * value.
1410                          */
1411                         xtmp128 = vmvnq_u32(vtstq_u64(olf, xtmp128));
1412                         ytmp128 = vmvnq_u32(vtstq_u64(olf, ytmp128));
1413
1414                         /* Change send mem address to an 8 byte offset when
1415                          * TSTMP is disabled.
1416                          */
1417                         sendmem01_w1 = vaddq_u64(sendmem01_w1,
1418                                                  vandq_u64(xtmp128, addr));
1419                         sendmem23_w1 = vaddq_u64(sendmem23_w1,
1420                                                  vandq_u64(ytmp128, addr));
1421                         /* Change send mem alg to SUB when TSTMP is disabled. */
1422                         sendmem01_w0 = vorrq_u64(sendmem01_w0,
1423                                                  vandq_u64(xtmp128, alg));
1424                         sendmem23_w0 = vorrq_u64(sendmem23_w0,
1425                                                  vandq_u64(ytmp128, alg));
1426
1427                         cmd3[0] = vzip1q_u64(sendmem01_w0, sendmem01_w1);
1428                         cmd3[1] = vzip2q_u64(sendmem01_w0, sendmem01_w1);
1429                         cmd3[2] = vzip1q_u64(sendmem23_w0, sendmem23_w1);
1430                         cmd3[3] = vzip2q_u64(sendmem23_w0, sendmem23_w1);
1431                 }
1432
1433                 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
1434                         /* Set don't free bit if reference count > 1 */
1435                         xmask01 = vdupq_n_u64(0);
1436                         xmask23 = xmask01;
1437
1438                         /* Move mbufs to iova */
1439                         mbuf0 = (uint64_t *)tx_pkts[0];
1440                         mbuf1 = (uint64_t *)tx_pkts[1];
1441                         mbuf2 = (uint64_t *)tx_pkts[2];
1442                         mbuf3 = (uint64_t *)tx_pkts[3];
1443
1444                         if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf0))
1445                                 vsetq_lane_u64(0x80000, xmask01, 0);
1446                         else
1447                                 __mempool_check_cookies(
1448                                         ((struct rte_mbuf *)mbuf0)->pool,
1449                                         (void **)&mbuf0, 1, 0);
1450
1451                         if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf1))
1452                                 vsetq_lane_u64(0x80000, xmask01, 1);
1453                         else
1454                                 __mempool_check_cookies(
1455                                         ((struct rte_mbuf *)mbuf1)->pool,
1456                                         (void **)&mbuf1, 1, 0);
1457
1458                         if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf2))
1459                                 vsetq_lane_u64(0x80000, xmask23, 0);
1460                         else
1461                                 __mempool_check_cookies(
1462                                         ((struct rte_mbuf *)mbuf2)->pool,
1463                                         (void **)&mbuf2, 1, 0);
1464
1465                         if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf3))
1466                                 vsetq_lane_u64(0x80000, xmask23, 1);
1467                         else
1468                                 __mempool_check_cookies(
1469                                         ((struct rte_mbuf *)mbuf3)->pool,
1470                                         (void **)&mbuf3, 1, 0);
1471                         senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
1472                         senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
1473                 } else {
1474                         /* Move mbufs to iova */
1475                         mbuf0 = (uint64_t *)tx_pkts[0];
1476                         mbuf1 = (uint64_t *)tx_pkts[1];
1477                         mbuf2 = (uint64_t *)tx_pkts[2];
1478                         mbuf3 = (uint64_t *)tx_pkts[3];
1479
1480                         /* Mark mempool object as "put" since
1481                          * it is freed by NIX
1482                          */
1483                         __mempool_check_cookies(
1484                                 ((struct rte_mbuf *)mbuf0)->pool,
1485                                 (void **)&mbuf0, 1, 0);
1486
1487                         __mempool_check_cookies(
1488                                 ((struct rte_mbuf *)mbuf1)->pool,
1489                                 (void **)&mbuf1, 1, 0);
1490
1491                         __mempool_check_cookies(
1492                                 ((struct rte_mbuf *)mbuf2)->pool,
1493                                 (void **)&mbuf2, 1, 0);
1494
1495                         __mempool_check_cookies(
1496                                 ((struct rte_mbuf *)mbuf3)->pool,
1497                                 (void **)&mbuf3, 1, 0);
1498                 }
1499
1500                 /* Create 4W cmd for 4 mbufs (sendhdr, sgdesc) */
1501                 cmd0[0] = vzip1q_u64(senddesc01_w0, senddesc01_w1);
1502                 cmd0[1] = vzip2q_u64(senddesc01_w0, senddesc01_w1);
1503                 cmd0[2] = vzip1q_u64(senddesc23_w0, senddesc23_w1);
1504                 cmd0[3] = vzip2q_u64(senddesc23_w0, senddesc23_w1);
1505
1506                 cmd1[0] = vzip1q_u64(sgdesc01_w0, sgdesc01_w1);
1507                 cmd1[1] = vzip2q_u64(sgdesc01_w0, sgdesc01_w1);
1508                 cmd1[2] = vzip1q_u64(sgdesc23_w0, sgdesc23_w1);
1509                 cmd1[3] = vzip2q_u64(sgdesc23_w0, sgdesc23_w1);
1510
1511                 if (flags & NIX_TX_NEED_EXT_HDR) {
1512                         cmd2[0] = vzip1q_u64(sendext01_w0, sendext01_w1);
1513                         cmd2[1] = vzip2q_u64(sendext01_w0, sendext01_w1);
1514                         cmd2[2] = vzip1q_u64(sendext23_w0, sendext23_w1);
1515                         cmd2[3] = vzip2q_u64(sendext23_w0, sendext23_w1);
1516                 }
1517
1518                 if (flags & NIX_TX_NEED_EXT_HDR) {
1519                         /* Store the prepared send desc to LMT lines */
1520                         if (flags & NIX_TX_OFFLOAD_TSTAMP_F) {
1521                                 vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[0]);
1522                                 vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[0]);
1523                                 vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[0]);
1524                                 vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd3[0]);
1525                                 vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd0[1]);
1526                                 vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd2[1]);
1527                                 vst1q_u64(LMT_OFF(laddr, lnum, 96), cmd1[1]);
1528                                 vst1q_u64(LMT_OFF(laddr, lnum, 112), cmd3[1]);
1529                                 lnum += 1;
1530                                 vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[2]);
1531                                 vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[2]);
1532                                 vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[2]);
1533                                 vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd3[2]);
1534                                 vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd0[3]);
1535                                 vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd2[3]);
1536                                 vst1q_u64(LMT_OFF(laddr, lnum, 96), cmd1[3]);
1537                                 vst1q_u64(LMT_OFF(laddr, lnum, 112), cmd3[3]);
1538                         } else {
1539                                 vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[0]);
1540                                 vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[0]);
1541                                 vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[0]);
1542                                 vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd0[1]);
1543                                 vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd2[1]);
1544                                 vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd1[1]);
1545                                 lnum += 1;
1546                                 vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[2]);
1547                                 vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd2[2]);
1548                                 vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd1[2]);
1549                                 vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd0[3]);
1550                                 vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd2[3]);
1551                                 vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd1[3]);
1552                         }
1553                         lnum += 1;
1554                 } else {
1555                         /* Store the prepared send desc to LMT lines */
1556                         vst1q_u64(LMT_OFF(laddr, lnum, 0), cmd0[0]);
1557                         vst1q_u64(LMT_OFF(laddr, lnum, 16), cmd1[0]);
1558                         vst1q_u64(LMT_OFF(laddr, lnum, 32), cmd0[1]);
1559                         vst1q_u64(LMT_OFF(laddr, lnum, 48), cmd1[1]);
1560                         vst1q_u64(LMT_OFF(laddr, lnum, 64), cmd0[2]);
1561                         vst1q_u64(LMT_OFF(laddr, lnum, 80), cmd1[2]);
1562                         vst1q_u64(LMT_OFF(laddr, lnum, 96), cmd0[3]);
1563                         vst1q_u64(LMT_OFF(laddr, lnum, 112), cmd1[3]);
1564                         lnum += 1;
1565                 }
1566
1567                 tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP;
1568         }
1569
1570         /* Trigger LMTST */
1571         if (lnum > 16) {
1572                 data = cn10k_nix_tx_steor_vec_data(flags);
1573                 pa = io_addr | (data & 0x7) << 4;
1574                 data &= ~0x7ULL;
1575                 data |= (15ULL << 12);
1576                 data |= (uint64_t)lmt_id;
1577
1578                 /* STEOR0 */
1579                 roc_lmt_submit_steorl(data, pa);
1580
1581                 data = cn10k_nix_tx_steor_vec_data(flags);
1582                 pa = io_addr | (data & 0x7) << 4;
1583                 data &= ~0x7ULL;
1584                 data |= ((uint64_t)(lnum - 17)) << 12;
1585                 data |= (uint64_t)(lmt_id + 16);
1586
1587                 /* STEOR1 */
1588                 roc_lmt_submit_steorl(data, pa);
1589         } else if (lnum) {
1590                 data = cn10k_nix_tx_steor_vec_data(flags);
1591                 pa = io_addr | (data & 0x7) << 4;
1592                 data &= ~0x7ULL;
1593                 data |= ((uint64_t)(lnum - 1)) << 12;
1594                 data |= lmt_id;
1595
1596                 /* STEOR0 */
1597                 roc_lmt_submit_steorl(data, pa);
1598         }
1599
1600         left -= burst;
1601         rte_io_wmb();
1602         if (left)
1603                 goto again;
1604
1605         if (unlikely(scalar))
1606                 pkts += cn10k_nix_xmit_pkts(tx_queue, tx_pkts, scalar, cmd,
1607                                             flags);
1608
1609         return pkts;
1610 }
1611
1612 #else
1613 static __rte_always_inline uint16_t
1614 cn10k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
1615                            uint16_t pkts, uint64_t *cmd, const uint16_t flags)
1616 {
1617         RTE_SET_USED(tx_queue);
1618         RTE_SET_USED(tx_pkts);
1619         RTE_SET_USED(pkts);
1620         RTE_SET_USED(cmd);
1621         RTE_SET_USED(flags);
1622         return 0;
1623 }
1624 #endif
1625
1626 #define L3L4CSUM_F   NIX_TX_OFFLOAD_L3_L4_CSUM_F
1627 #define OL3OL4CSUM_F NIX_TX_OFFLOAD_OL3_OL4_CSUM_F
1628 #define VLAN_F       NIX_TX_OFFLOAD_VLAN_QINQ_F
1629 #define NOFF_F       NIX_TX_OFFLOAD_MBUF_NOFF_F
1630 #define TSO_F        NIX_TX_OFFLOAD_TSO_F
1631 #define TSP_F        NIX_TX_OFFLOAD_TSTAMP_F
1632
1633 /* [TSP] [TSO] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */
1634 #define NIX_TX_FASTPATH_MODES                                           \
1635 T(no_offload,                           0, 0, 0, 0, 0, 0,       4,      \
1636                 NIX_TX_OFFLOAD_NONE)                                    \
1637 T(l3l4csum,                             0, 0, 0, 0, 0, 1,       4,      \
1638                 L3L4CSUM_F)                                             \
1639 T(ol3ol4csum,                           0, 0, 0, 0, 1, 0,       4,      \
1640                 OL3OL4CSUM_F)                                           \
1641 T(ol3ol4csum_l3l4csum,                  0, 0, 0, 0, 1, 1,       4,      \
1642                 OL3OL4CSUM_F | L3L4CSUM_F)                              \
1643 T(vlan,                                 0, 0, 0, 1, 0, 0,       6,      \
1644                 VLAN_F)                                                 \
1645 T(vlan_l3l4csum,                        0, 0, 0, 1, 0, 1,       6,      \
1646                 VLAN_F | L3L4CSUM_F)                                    \
1647 T(vlan_ol3ol4csum,                      0, 0, 0, 1, 1, 0,       6,      \
1648                 VLAN_F | OL3OL4CSUM_F)                                  \
1649 T(vlan_ol3ol4csum_l3l4csum,             0, 0, 0, 1, 1, 1,       6,      \
1650                 VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)                     \
1651 T(noff,                                 0, 0, 1, 0, 0, 0,       4,      \
1652                 NOFF_F)                                                 \
1653 T(noff_l3l4csum,                        0, 0, 1, 0, 0, 1,       4,      \
1654                 NOFF_F | L3L4CSUM_F)                                    \
1655 T(noff_ol3ol4csum,                      0, 0, 1, 0, 1, 0,       4,      \
1656                 NOFF_F | OL3OL4CSUM_F)                                  \
1657 T(noff_ol3ol4csum_l3l4csum,             0, 0, 1, 0, 1, 1,       4,      \
1658                 NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F)                     \
1659 T(noff_vlan,                            0, 0, 1, 1, 0, 0,       6,      \
1660                 NOFF_F | VLAN_F)                                        \
1661 T(noff_vlan_l3l4csum,                   0, 0, 1, 1, 0, 1,       6,      \
1662                 NOFF_F | VLAN_F | L3L4CSUM_F)                           \
1663 T(noff_vlan_ol3ol4csum,                 0, 0, 1, 1, 1, 0,       6,      \
1664                 NOFF_F | VLAN_F | OL3OL4CSUM_F)                         \
1665 T(noff_vlan_ol3ol4csum_l3l4csum,        0, 0, 1, 1, 1, 1,       6,      \
1666                 NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)            \
1667 T(tso,                                  0, 1, 0, 0, 0, 0,       6,      \
1668                 TSO_F)                                                  \
1669 T(tso_l3l4csum,                         0, 1, 0, 0, 0, 1,       6,      \
1670                 TSO_F | L3L4CSUM_F)                                     \
1671 T(tso_ol3ol4csum,                       0, 1, 0, 0, 1, 0,       6,      \
1672                 TSO_F | OL3OL4CSUM_F)                                   \
1673 T(tso_ol3ol4csum_l3l4csum,              0, 1, 0, 0, 1, 1,       6,      \
1674                 TSO_F | OL3OL4CSUM_F | L3L4CSUM_F)                      \
1675 T(tso_vlan,                             0, 1, 0, 1, 0, 0,       6,      \
1676                 TSO_F | VLAN_F)                                         \
1677 T(tso_vlan_l3l4csum,                    0, 1, 0, 1, 0, 1,       6,      \
1678                 TSO_F | VLAN_F | L3L4CSUM_F)                            \
1679 T(tso_vlan_ol3ol4csum,                  0, 1, 0, 1, 1, 0,       6,      \
1680                 TSO_F | VLAN_F | OL3OL4CSUM_F)                          \
1681 T(tso_vlan_ol3ol4csum_l3l4csum,         0, 1, 0, 1, 1, 1,       6,      \
1682                 TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)             \
1683 T(tso_noff,                             0, 1, 1, 0, 0, 0,       6,      \
1684                 TSO_F | NOFF_F)                                         \
1685 T(tso_noff_l3l4csum,                    0, 1, 1, 0, 0, 1,       6,      \
1686                 TSO_F | NOFF_F | L3L4CSUM_F)                            \
1687 T(tso_noff_ol3ol4csum,                  0, 1, 1, 0, 1, 0,       6,      \
1688                 TSO_F | NOFF_F | OL3OL4CSUM_F)                          \
1689 T(tso_noff_ol3ol4csum_l3l4csum,         0, 1, 1, 0, 1, 1,       6,      \
1690                 TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F)             \
1691 T(tso_noff_vlan,                        0, 1, 1, 1, 0, 0,       6,      \
1692                 TSO_F | NOFF_F | VLAN_F)                                \
1693 T(tso_noff_vlan_l3l4csum,               0, 1, 1, 1, 0, 1,       6,      \
1694                 TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F)                   \
1695 T(tso_noff_vlan_ol3ol4csum,             0, 1, 1, 1, 1, 0,       6,      \
1696                 TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F)                 \
1697 T(tso_noff_vlan_ol3ol4csum_l3l4csum,    0, 1, 1, 1, 1, 1,       6,      \
1698                 TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)    \
1699 T(ts,                                   1, 0, 0, 0, 0, 0,       8,      \
1700                 TSP_F)                                                  \
1701 T(ts_l3l4csum,                          1, 0, 0, 0, 0, 1,       8,      \
1702                 TSP_F | L3L4CSUM_F)                                     \
1703 T(ts_ol3ol4csum,                        1, 0, 0, 0, 1, 0,       8,      \
1704                 TSP_F | OL3OL4CSUM_F)                                   \
1705 T(ts_ol3ol4csum_l3l4csum,               1, 0, 0, 0, 1, 1,       8,      \
1706                 TSP_F | OL3OL4CSUM_F | L3L4CSUM_F)                      \
1707 T(ts_vlan,                              1, 0, 0, 1, 0, 0,       8,      \
1708                 TSP_F | VLAN_F)                                         \
1709 T(ts_vlan_l3l4csum,                     1, 0, 0, 1, 0, 1,       8,      \
1710                 TSP_F | VLAN_F | L3L4CSUM_F)                            \
1711 T(ts_vlan_ol3ol4csum,                   1, 0, 0, 1, 1, 0,       8,      \
1712                 TSP_F | VLAN_F | OL3OL4CSUM_F)                          \
1713 T(ts_vlan_ol3ol4csum_l3l4csum,          1, 0, 0, 1, 1, 1,       8,      \
1714                 TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)             \
1715 T(ts_noff,                              1, 0, 1, 0, 0, 0,       8,      \
1716                 TSP_F | NOFF_F)                                         \
1717 T(ts_noff_l3l4csum,                     1, 0, 1, 0, 0, 1,       8,      \
1718                 TSP_F | NOFF_F | L3L4CSUM_F)                            \
1719 T(ts_noff_ol3ol4csum,                   1, 0, 1, 0, 1, 0,       8,      \
1720                 TSP_F | NOFF_F | OL3OL4CSUM_F)                          \
1721 T(ts_noff_ol3ol4csum_l3l4csum,          1, 0, 1, 0, 1, 1,       8,      \
1722                 TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F)             \
1723 T(ts_noff_vlan,                         1, 0, 1, 1, 0, 0,       8,      \
1724                 TSP_F | NOFF_F | VLAN_F)                                \
1725 T(ts_noff_vlan_l3l4csum,                1, 0, 1, 1, 0, 1,       8,      \
1726                 TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F)                   \
1727 T(ts_noff_vlan_ol3ol4csum,              1, 0, 1, 1, 1, 0,       8,      \
1728                 TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F)                 \
1729 T(ts_noff_vlan_ol3ol4csum_l3l4csum,     1, 0, 1, 1, 1, 1,       8,      \
1730                 TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)    \
1731 T(ts_tso,                               1, 1, 0, 0, 0, 0,       8,      \
1732                 TSP_F | TSO_F)                                          \
1733 T(ts_tso_l3l4csum,                      1, 1, 0, 0, 0, 1,       8,      \
1734                 TSP_F | TSO_F | L3L4CSUM_F)                             \
1735 T(ts_tso_ol3ol4csum,                    1, 1, 0, 0, 1, 0,       8,      \
1736                 TSP_F | TSO_F | OL3OL4CSUM_F)                           \
1737 T(ts_tso_ol3ol4csum_l3l4csum,           1, 1, 0, 0, 1, 1,       8,      \
1738                 TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F)              \
1739 T(ts_tso_vlan,                          1, 1, 0, 1, 0, 0,       8,      \
1740                 TSP_F | TSO_F | VLAN_F)                                 \
1741 T(ts_tso_vlan_l3l4csum,                 1, 1, 0, 1, 0, 1,       8,      \
1742                 TSP_F | TSO_F | VLAN_F | L3L4CSUM_F)                    \
1743 T(ts_tso_vlan_ol3ol4csum,               1, 1, 0, 1, 1, 0,       8,      \
1744                 TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F)                  \
1745 T(ts_tso_vlan_ol3ol4csum_l3l4csum,      1, 1, 0, 1, 1, 1,       8,      \
1746                 TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)     \
1747 T(ts_tso_noff,                          1, 1, 1, 0, 0, 0,       8,      \
1748                 TSP_F | TSO_F | NOFF_F)                                 \
1749 T(ts_tso_noff_l3l4csum,                 1, 1, 1, 0, 0, 1,       8,      \
1750                 TSP_F | TSO_F | NOFF_F | L3L4CSUM_F)                    \
1751 T(ts_tso_noff_ol3ol4csum,               1, 1, 1, 0, 1, 0,       8,      \
1752                 TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F)                  \
1753 T(ts_tso_noff_ol3ol4csum_l3l4csum,      1, 1, 1, 0, 1, 1,       8,      \
1754                 TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F)     \
1755 T(ts_tso_noff_vlan,                     1, 1, 1, 1, 0, 0,       8,      \
1756                 TSP_F | TSO_F | NOFF_F | VLAN_F)                        \
1757 T(ts_tso_noff_vlan_l3l4csum,            1, 1, 1, 1, 0, 1,       8,      \
1758                 TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F)           \
1759 T(ts_tso_noff_vlan_ol3ol4csum,          1, 1, 1, 1, 1, 0,       8,      \
1760                 TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F)         \
1761 T(ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 1, 1,       8,      \
1762                 TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)
1763
1764 #define T(name, f5, f4, f3, f2, f1, f0, sz, flags)                             \
1765         uint16_t __rte_noinline __rte_hot cn10k_nix_xmit_pkts_##name(          \
1766                 void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts);     \
1767                                                                                \
1768         uint16_t __rte_noinline __rte_hot cn10k_nix_xmit_pkts_mseg_##name(     \
1769                 void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts);     \
1770                                                                                \
1771         uint16_t __rte_noinline __rte_hot cn10k_nix_xmit_pkts_vec_##name(      \
1772                 void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts);
1773
1774 NIX_TX_FASTPATH_MODES
1775 #undef T
1776
1777 #endif /* __CN10K_TX_H__ */