drivers/net/cnxk/cn10k_rx.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4 #ifndef __CN10K_RX_H__
   5 #define __CN10K_RX_H__
   6
   7 #include <rte_ether.h>
   8 #include <rte_vect.h>
   9
  10 #include <cnxk_ethdev.h>
  11
  12 #define NIX_RX_OFFLOAD_NONE          (0)
  13 #define NIX_RX_OFFLOAD_RSS_F         BIT(0)
  14 #define NIX_RX_OFFLOAD_PTYPE_F       BIT(1)
  15 #define NIX_RX_OFFLOAD_CHECKSUM_F    BIT(2)
  16 #define NIX_RX_OFFLOAD_MARK_UPDATE_F BIT(3)
  17 #define NIX_RX_OFFLOAD_TSTAMP_F      BIT(4)
  18 #define NIX_RX_OFFLOAD_VLAN_STRIP_F  BIT(5)
  19 #define NIX_RX_OFFLOAD_SECURITY_F    BIT(6)
  20 #define NIX_RX_OFFLOAD_MAX           (NIX_RX_OFFLOAD_SECURITY_F << 1)
  21
  22 /* Flags to control cqe_to_mbuf conversion function.
  23  * Defining it from backwards to denote its been
  24  * not used as offload flags to pick function
  25  */
  26 #define NIX_RX_REAS_F      BIT(12)
  27 #define NIX_RX_VWQE_F      BIT(13)
  28 #define NIX_RX_MULTI_SEG_F BIT(14)
  29 #define CPT_RX_WQE_F       BIT(15)
  30
  31 #define CNXK_NIX_CQ_ENTRY_SZ 128
  32 #define NIX_DESCS_PER_LOOP   4
  33 #define CQE_CAST(x)          ((struct nix_cqe_hdr_s *)(x))
  34 #define CQE_SZ(x)            ((x) * CNXK_NIX_CQ_ENTRY_SZ)
  35
  36 #define CQE_PTR_OFF(b, i, o, f)                                                \
  37         (((f) & NIX_RX_VWQE_F) ?                                               \
  38                        (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) + (o)) : \
  39                        (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) + (o)))
  40 #define CQE_PTR_DIFF(b, i, o, f)                                               \
  41         (((f) & NIX_RX_VWQE_F) ?                                               \
  42                  (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) - (o)) :       \
  43                        (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) - (o)))
  44
  45 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
  46 static inline void
  47 nix_mbuf_validate_next(struct rte_mbuf *m)
  48 {
  49         if (m->nb_segs == 1 && m->next) {
  50                 rte_panic("mbuf->next[%p] valid when mbuf->nb_segs is %d",
  51                         m->next, m->nb_segs);
  52         }
  53 }
  54 #else
  55 static inline void
  56 nix_mbuf_validate_next(struct rte_mbuf *m)
  57 {
  58         RTE_SET_USED(m);
  59 }
  60 #endif
  61
  62 #define NIX_RX_SEC_REASSEMBLY_F \
  63         (NIX_RX_REAS_F | NIX_RX_OFFLOAD_SECURITY_F)
  64
  65 static inline rte_eth_ip_reassembly_dynfield_t *
  66 cnxk_ip_reassembly_dynfield(struct rte_mbuf *mbuf,
  67                 int ip_reassembly_dynfield_offset)
  68 {
  69         return RTE_MBUF_DYNFIELD(mbuf, ip_reassembly_dynfield_offset,
  70                                  rte_eth_ip_reassembly_dynfield_t *);
  71 }
  72
  73 union mbuf_initializer {
  74         struct {
  75                 uint16_t data_off;
  76                 uint16_t refcnt;
  77                 uint16_t nb_segs;
  78                 uint16_t port;
  79         } fields;
  80         uint64_t value;
  81 };
  82
  83 static __rte_always_inline uint64_t
  84 nix_clear_data_off(uint64_t oldval)
  85 {
  86         union mbuf_initializer mbuf_init = {.value = oldval};
  87
  88         mbuf_init.fields.data_off = 0;
  89         return mbuf_init.value;
  90 }
  91
  92 static __rte_always_inline struct rte_mbuf *
  93 nix_get_mbuf_from_cqe(void *cq, const uint64_t data_off)
  94 {
  95         rte_iova_t buff;
  96
  97         /* Skip CQE, NIX_RX_PARSE_S and SG HDR(9 DWORDs) and peek buff addr */
  98         buff = *((rte_iova_t *)((uint64_t *)cq + 9));
  99         return (struct rte_mbuf *)(buff - data_off);
 100 }
 101
 102 static __rte_always_inline void
 103 nix_sec_flush_meta_burst(uint16_t lmt_id, uint64_t data, uint16_t lnum,
 104                          uintptr_t aura_handle)
 105 {
 106         uint64_t pa;
 107
 108         /* Prepare PA and Data */
 109         pa = roc_npa_aura_handle_to_base(aura_handle) + NPA_LF_AURA_BATCH_FREE0;
 110         pa |= ((data & 0x7) << 4);
 111
 112         data >>= 3;
 113         data <<= 19;
 114         data |= (uint64_t)lmt_id;
 115         data |= (uint64_t)(lnum - 1) << 12;
 116
 117         roc_lmt_submit_steorl(data, pa);
 118 }
 119
 120 static __rte_always_inline void
 121 nix_sec_flush_meta(uintptr_t laddr, uint16_t lmt_id, uint8_t loff,
 122                    uintptr_t aura_handle)
 123 {
 124         uint64_t pa;
 125
 126         /* laddr is pointing to first pointer */
 127         laddr -= 8;
 128
 129         /* Trigger free either on lmtline full or different aura handle */
 130         pa = roc_npa_aura_handle_to_base(aura_handle) + NPA_LF_AURA_BATCH_FREE0;
 131
 132         /* Update aura handle */
 133         *(uint64_t *)laddr = (((uint64_t)(loff & 0x1) << 32) |
 134                               roc_npa_aura_handle_to_aura(aura_handle));
 135
 136         pa |= ((uint64_t)(loff >> 1) << 4);
 137         roc_lmt_submit_steorl(lmt_id, pa);
 138 }
 139
 140 static struct rte_mbuf *
 141 nix_sec_attach_frags(const struct cpt_parse_hdr_s *hdr,
 142                      struct cn10k_inb_priv_data *inb_priv,
 143                      const uint64_t mbuf_init)
 144 {
 145         struct rte_mbuf *head, *mbuf, *mbuf_prev;
 146         uint32_t offset = hdr->w2.fi_offset;
 147         union nix_rx_parse_u *frag_rx;
 148         struct cpt_frag_info_s *finfo;
 149         uint64_t *frag_ptr = NULL;
 150         uint64_t ol_flags;
 151         uint16_t frag_size;
 152         uint16_t rlen;
 153         uint64_t *wqe;
 154         int off;
 155
 156         off = inb_priv->reass_dynfield_off;
 157         ol_flags = BIT_ULL(inb_priv->reass_dynflag_bit);
 158         ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD;
 159
 160         /* offset of 0 implies 256B, otherwise it implies offset*8B */
 161         offset = (((offset - 1) & 0x1f) + 1) * 8;
 162         finfo = RTE_PTR_ADD(hdr, offset + hdr->w2.fi_pad);
 163
 164         /* Frag-0: */
 165         wqe = (uint64_t *)(rte_be_to_cpu_64(hdr->wqe_ptr));
 166         rlen = ((*(wqe + 10)) >> 16) & 0xFFFF;
 167
 168         frag_rx = (union nix_rx_parse_u *)(wqe + 1);
 169         frag_size = rlen + frag_rx->lcptr - frag_rx->laptr;
 170         frag_rx->pkt_lenm1 = frag_size - 1;
 171
 172         mbuf = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf));
 173         *(uint64_t *)(&mbuf->rearm_data) = mbuf_init;
 174         mbuf->data_len = frag_size;
 175         mbuf->pkt_len = frag_size;
 176         mbuf->ol_flags = ol_flags;
 177         mbuf->next = NULL;
 178         head = mbuf;
 179         mbuf_prev = mbuf;
 180         /* Update dynamic field with userdata */
 181         *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata;
 182
 183         cnxk_ip_reassembly_dynfield(head, off)->nb_frags = hdr->w0.num_frags - 1;
 184         cnxk_ip_reassembly_dynfield(head, off)->next_frag = NULL;
 185
 186         /* Frag-1: */
 187         if (hdr->w0.num_frags > 1) {
 188                 wqe = (uint64_t *)(rte_be_to_cpu_64(hdr->frag1_wqe_ptr));
 189                 rlen = ((*(wqe + 10)) >> 16) & 0xFFFF;
 190
 191                 frag_rx = (union nix_rx_parse_u *)(wqe + 1);
 192                 frag_size = rlen + frag_rx->lcptr - frag_rx->laptr;
 193                 frag_rx->pkt_lenm1 = frag_size - 1;
 194
 195                 mbuf = (struct rte_mbuf *)((uintptr_t)wqe -
 196                                 sizeof(struct rte_mbuf));
 197                 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init;
 198                 mbuf->data_len = frag_size;
 199                 mbuf->pkt_len = frag_size;
 200                 mbuf->ol_flags = ol_flags;
 201                 mbuf->next = NULL;
 202
 203                 /* Update dynamic field with userdata */
 204                 *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata;
 205
 206                 cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags =
 207                         hdr->w0.num_frags - 2;
 208                 cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = NULL;
 209                 cnxk_ip_reassembly_dynfield(mbuf_prev, off)->next_frag = mbuf;
 210                 mbuf_prev = mbuf;
 211         }
 212
 213         /* Frag-2: */
 214         if (hdr->w0.num_frags > 2) {
 215                 frag_ptr = (uint64_t *)(finfo + 1);
 216                 wqe = (uint64_t *)(rte_be_to_cpu_64(*frag_ptr));
 217                 rlen = ((*(wqe + 10)) >> 16) & 0xFFFF;
 218
 219                 frag_rx = (union nix_rx_parse_u *)(wqe + 1);
 220                 frag_size = rlen + frag_rx->lcptr - frag_rx->laptr;
 221                 frag_rx->pkt_lenm1 = frag_size - 1;
 222
 223                 mbuf = (struct rte_mbuf *)((uintptr_t)wqe -
 224                                 sizeof(struct rte_mbuf));
 225                 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init;
 226                 mbuf->data_len = frag_size;
 227                 mbuf->pkt_len = frag_size;
 228                 mbuf->ol_flags = ol_flags;
 229                 mbuf->next = NULL;
 230
 231                 /* Update dynamic field with userdata */
 232                 *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata;
 233
 234                 cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags =
 235                         hdr->w0.num_frags - 3;
 236                 cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = NULL;
 237                 cnxk_ip_reassembly_dynfield(mbuf_prev, off)->next_frag = mbuf;
 238                 mbuf_prev = mbuf;
 239         }
 240
 241         /* Frag-3: */
 242         if (hdr->w0.num_frags > 3) {
 243                 wqe = (uint64_t *)(rte_be_to_cpu_64(*(frag_ptr + 1)));
 244                 rlen = ((*(wqe + 10)) >> 16) & 0xFFFF;
 245
 246                 frag_rx = (union nix_rx_parse_u *)(wqe + 1);
 247                 frag_size = rlen + frag_rx->lcptr - frag_rx->laptr;
 248                 frag_rx->pkt_lenm1 = frag_size - 1;
 249
 250                 mbuf = (struct rte_mbuf *)((uintptr_t)wqe -
 251                                 sizeof(struct rte_mbuf));
 252                 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init;
 253                 mbuf->data_len = frag_size;
 254                 mbuf->pkt_len = frag_size;
 255                 mbuf->ol_flags = ol_flags;
 256                 mbuf->next = NULL;
 257
 258                 /* Update dynamic field with userdata */
 259                 *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata;
 260
 261                 cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags =
 262                         hdr->w0.num_frags - 4;
 263                 cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = NULL;
 264                 cnxk_ip_reassembly_dynfield(mbuf_prev, off)->next_frag = mbuf;
 265         }
 266         return head;
 267 }
 268
 269 static struct rte_mbuf *
 270 nix_sec_reassemble_frags(const struct cpt_parse_hdr_s *hdr, uint64_t cq_w1,
 271                         uint64_t cq_w5, uint64_t mbuf_init)
 272 {
 273         uint32_t fragx_sum, pkt_hdr_len, l3_hdr_size;
 274         uint32_t offset = hdr->w2.fi_offset;
 275         union nix_rx_parse_u *inner_rx;
 276         uint16_t rlen, data_off, b_off;
 277         union nix_rx_parse_u *frag_rx;
 278         struct cpt_frag_info_s *finfo;
 279         struct rte_mbuf *head, *mbuf;
 280         uint64_t *frag_ptr = NULL;
 281         rte_iova_t *inner_iova;
 282         uint16_t frag_size;
 283         uint64_t *wqe;
 284
 285         /* Base data offset */
 286         b_off = mbuf_init & 0xFFFFUL;
 287         mbuf_init &= ~0xFFFFUL;
 288
 289         /* offset of 0 implies 256B, otherwise it implies offset*8B */
 290         offset = (((offset - 1) & 0x1f) + 1) * 8;
 291         finfo = RTE_PTR_ADD(hdr, offset + hdr->w2.fi_pad);
 292
 293         /* Frag-0: */
 294         wqe = (uint64_t *)rte_be_to_cpu_64(hdr->wqe_ptr);
 295         inner_rx = (union nix_rx_parse_u *)(wqe + 1);
 296         inner_iova = (rte_iova_t *)*(wqe + 9);
 297
 298         /* Update only the upper 28-bits from meta pkt parse info */
 299         *((uint64_t *)inner_rx) = ((*((uint64_t *)inner_rx) & ((1ULL << 36) - 1)) |
 300                                 (cq_w1 & ~((1ULL << 36) - 1)));
 301
 302         rlen = ((*(wqe + 10)) >> 16) & 0xFFFF;
 303         frag_size = rlen + ((cq_w5 >> 16) & 0xFF) - (cq_w5 & 0xFF);
 304         fragx_sum = rte_be_to_cpu_16(finfo->w1.frag_size0);
 305         pkt_hdr_len = frag_size - fragx_sum;
 306
 307         mbuf = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf));
 308         *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | b_off;
 309         mbuf->data_len = frag_size;
 310         head = mbuf;
 311
 312         if (inner_rx->lctype == NPC_LT_LC_IP) {
 313                 struct rte_ipv4_hdr *hdr = (struct rte_ipv4_hdr *)
 314                                 RTE_PTR_ADD(inner_iova, inner_rx->lcptr);
 315
 316                 l3_hdr_size = (hdr->version_ihl & 0xf) << 2;
 317         } else {
 318                 struct rte_ipv6_hdr *hdr = (struct rte_ipv6_hdr *)
 319                                 RTE_PTR_ADD(inner_iova, inner_rx->lcptr);
 320                 size_t ext_len = sizeof(struct rte_ipv6_hdr);
 321                 uint8_t *nxt_hdr = (uint8_t *)hdr;
 322                 int nh = hdr->proto;
 323
 324                 l3_hdr_size = 0;
 325                 while (nh != -EINVAL) {
 326                         nxt_hdr += ext_len;
 327                         l3_hdr_size += ext_len;
 328                         nh = rte_ipv6_get_next_ext(nxt_hdr, nh, &ext_len);
 329                 }
 330         }
 331
 332         /* Frag-1: */
 333         wqe = (uint64_t *)(rte_be_to_cpu_64(hdr->frag1_wqe_ptr));
 334         frag_size = rte_be_to_cpu_16(finfo->w1.frag_size1);
 335         frag_rx = (union nix_rx_parse_u *)(wqe + 1);
 336
 337         mbuf->next = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf));
 338         mbuf = mbuf->next;
 339         data_off = b_off + frag_rx->lcptr + l3_hdr_size;
 340         *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off;
 341         mbuf->data_len = frag_size;
 342         fragx_sum += frag_size;
 343
 344         /* Frag-2: */
 345         if (hdr->w0.num_frags > 2) {
 346                 frag_ptr = (uint64_t *)(finfo + 1);
 347                 wqe = (uint64_t *)(rte_be_to_cpu_64(*frag_ptr));
 348                 frag_size = rte_be_to_cpu_16(finfo->w1.frag_size2);
 349                 frag_rx = (union nix_rx_parse_u *)(wqe + 1);
 350
 351                 mbuf->next = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf));
 352                 mbuf = mbuf->next;
 353                 data_off = b_off + frag_rx->lcptr + l3_hdr_size;
 354                 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off;
 355                 mbuf->data_len = frag_size;
 356                 fragx_sum += frag_size;
 357         }
 358
 359         /* Frag-3: */
 360         if (hdr->w0.num_frags > 3) {
 361                 wqe = (uint64_t *)(rte_be_to_cpu_64(*(frag_ptr + 1)));
 362                 frag_size = rte_be_to_cpu_16(finfo->w1.frag_size3);
 363                 frag_rx = (union nix_rx_parse_u *)(wqe + 1);
 364
 365                 mbuf->next = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf));
 366                 mbuf = mbuf->next;
 367                 data_off = b_off + frag_rx->lcptr + l3_hdr_size;
 368                 *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off;
 369                 mbuf->data_len = frag_size;
 370                 fragx_sum += frag_size;
 371         }
 372
 373         if (inner_rx->lctype == NPC_LT_LC_IP) {
 374                 struct rte_ipv4_hdr *hdr = (struct rte_ipv4_hdr *)
 375                                 RTE_PTR_ADD(inner_iova, inner_rx->lcptr);
 376
 377                 hdr->fragment_offset = 0;
 378                 hdr->total_length = rte_cpu_to_be_16(fragx_sum + l3_hdr_size);
 379                 hdr->hdr_checksum = 0;
 380                 hdr->hdr_checksum = rte_ipv4_cksum(hdr);
 381
 382                 inner_rx->pkt_lenm1 = pkt_hdr_len + fragx_sum - 1;
 383         } else {
 384                 /* Remove the frag header by moving header 8 bytes forward */
 385                 struct rte_ipv6_hdr *hdr = (struct rte_ipv6_hdr *)
 386                                 RTE_PTR_ADD(inner_iova, inner_rx->lcptr);
 387
 388                 hdr->payload_len = rte_cpu_to_be_16(fragx_sum + l3_hdr_size -
 389                                         8 - sizeof(struct rte_ipv6_hdr));
 390
 391                 rte_memcpy(rte_pktmbuf_mtod_offset(head, void *, 8),
 392                            rte_pktmbuf_mtod(head, void *),
 393                            inner_rx->lcptr + sizeof(struct rte_ipv6_hdr));
 394
 395                 inner_rx->pkt_lenm1 = pkt_hdr_len + fragx_sum - 8 - 1;
 396                 head->data_len -= 8;
 397                 head->data_off += 8;
 398         }
 399         mbuf->next = NULL;
 400         head->pkt_len = inner_rx->pkt_lenm1 + 1;
 401         head->nb_segs = hdr->w0.num_frags;
 402
 403         return head;
 404 }
 405
 406 static __rte_always_inline struct rte_mbuf *
 407 nix_sec_meta_to_mbuf_sc(uint64_t cq_w1, uint64_t cq_w5, const uint64_t sa_base,
 408                         uintptr_t laddr, uint8_t *loff, struct rte_mbuf *mbuf,
 409                         uint16_t data_off, const uint16_t flags,
 410                         const uint64_t mbuf_init)
 411 {
 412         const void *__p = (void *)((uintptr_t)mbuf + (uint16_t)data_off);
 413         const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)__p;
 414         struct cn10k_inb_priv_data *inb_priv;
 415         struct rte_mbuf *inner = NULL;
 416         uint64_t res_w1;
 417         uint32_t sa_idx;
 418         uint16_t uc_cc;
 419         uint32_t len;
 420         void *inb_sa;
 421         uint64_t w0;
 422
 423         if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11))) {
 424                 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */
 425                 w0 = hdr->w0.u64;
 426                 sa_idx = w0 >> 32;
 427
 428                 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx);
 429                 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa);
 430
 431                 if (!hdr->w0.num_frags) {
 432                         /* No Reassembly or inbound error */
 433                         inner = (struct rte_mbuf *)
 434                                 (rte_be_to_cpu_64(hdr->wqe_ptr) -
 435                                  sizeof(struct rte_mbuf));
 436
 437                         /* Update dynamic field with userdata */
 438                         *rte_security_dynfield(inner) =
 439                                 (uint64_t)inb_priv->userdata;
 440
 441                         /* CPT result(struct cpt_cn10k_res_s) is at
 442                          * after first IOVA in meta
 443                          */
 444                         res_w1 = *((uint64_t *)(&inner[1]) + 10);
 445                         uc_cc = res_w1 & 0xFF;
 446
 447                         /* Calculate inner packet length */
 448                         len = ((res_w1 >> 16) & 0xFFFF) + hdr->w2.il3_off -
 449                                 sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7);
 450                         inner->pkt_len = len;
 451                         inner->data_len = len;
 452                         *(uint64_t *)(&inner->rearm_data) = mbuf_init;
 453
 454                         inner->ol_flags = ((uc_cc == CPT_COMP_WARN) ?
 455                                            RTE_MBUF_F_RX_SEC_OFFLOAD :
 456                                            (RTE_MBUF_F_RX_SEC_OFFLOAD |
 457                                             RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED));
 458                 } else if (!(hdr->w0.err_sum) && !(hdr->w0.reas_sts)) {
 459                         /* Reassembly success */
 460                         inner = nix_sec_reassemble_frags(hdr, cq_w1, cq_w5,
 461                                                          mbuf_init);
 462
 463                         /* Update dynamic field with userdata */
 464                         *rte_security_dynfield(inner) =
 465                                 (uint64_t)inb_priv->userdata;
 466
 467                         /* Assume success */
 468                         inner->ol_flags = RTE_MBUF_F_RX_SEC_OFFLOAD;
 469                 } else {
 470                         /* Reassembly failure */
 471                         inner = nix_sec_attach_frags(hdr, inb_priv, mbuf_init);
 472                 }
 473
 474                 /* Store meta in lmtline to free
 475                  * Assume all meta's from same aura.
 476                  */
 477                 *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf;
 478                 *loff = *loff + 1;
 479
 480                 return inner;
 481         } else if (cq_w1 & BIT(11)) {
 482                 inner = (struct rte_mbuf *)(rte_be_to_cpu_64(hdr->wqe_ptr) -
 483                                             sizeof(struct rte_mbuf));
 484
 485                 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */
 486                 w0 = hdr->w0.u64;
 487                 sa_idx = w0 >> 32;
 488
 489                 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx);
 490                 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa);
 491
 492                 /* Update dynamic field with userdata */
 493                 *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata;
 494
 495                 /* Update l2 hdr length first */
 496
 497                 /* CPT result(struct cpt_cn10k_res_s) is at
 498                  * after first IOVA in meta
 499                  */
 500                 res_w1 = *((uint64_t *)(&inner[1]) + 10);
 501                 uc_cc = res_w1 & 0xFF;
 502
 503                 /* Calculate inner packet length */
 504                 len = ((res_w1 >> 16) & 0xFFFF) + hdr->w2.il3_off -
 505                         sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7);
 506                 inner->pkt_len = len;
 507                 inner->data_len = len;
 508                 *(uint64_t *)(&inner->rearm_data) = mbuf_init;
 509
 510                 inner->ol_flags = ((uc_cc == CPT_COMP_WARN) ?
 511                                    RTE_MBUF_F_RX_SEC_OFFLOAD :
 512                                    (RTE_MBUF_F_RX_SEC_OFFLOAD |
 513                                     RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED));
 514
 515                 /* Store meta in lmtline to free
 516                  * Assume all meta's from same aura.
 517                  */
 518                 *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf;
 519                 *loff = *loff + 1;
 520
 521                 /* Mark meta mbuf as put */
 522                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 0);
 523
 524                 /* Mark inner mbuf as get */
 525                 RTE_MEMPOOL_CHECK_COOKIES(inner->pool, (void **)&inner, 1, 1);
 526
 527                 return inner;
 528         }
 529
 530         return mbuf;
 531 }
 532
 533 #if defined(RTE_ARCH_ARM64)
 534
 535 static __rte_always_inline struct rte_mbuf *
 536 nix_sec_meta_to_mbuf(uint64_t cq_w1, uint64_t cq_w5, uintptr_t sa_base,
 537                      uintptr_t laddr, uint8_t *loff, struct rte_mbuf *mbuf,
 538                      uint16_t data_off, uint8x16_t *rx_desc_field1,
 539                      uint64_t *ol_flags, const uint16_t flags,
 540                      uint64x2_t *rearm)
 541 {
 542         const void *__p = (void *)((uintptr_t)mbuf + (uint16_t)data_off);
 543         const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)__p;
 544         uint64_t mbuf_init = vgetq_lane_u64(*rearm, 0);
 545         struct cn10k_inb_priv_data *inb_priv;
 546         struct rte_mbuf *inner;
 547         uint64_t *sg, res_w1;
 548         uint32_t sa_idx;
 549         void *inb_sa;
 550         uint16_t len;
 551         uint64_t w0;
 552
 553         if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11))) {
 554                 w0 = hdr->w0.u64;
 555                 sa_idx = w0 >> 32;
 556
 557                 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */
 558                 w0 = hdr->w0.u64;
 559                 sa_idx = w0 >> 32;
 560
 561                 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx);
 562                 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa);
 563
 564                 /* Clear checksum flags */
 565                 *ol_flags &= ~(RTE_MBUF_F_RX_L4_CKSUM_MASK |
 566                                RTE_MBUF_F_RX_IP_CKSUM_MASK);
 567
 568                 if (!hdr->w0.num_frags) {
 569                         /* No Reassembly or inbound error */
 570                         inner = (struct rte_mbuf *)
 571                                 (rte_be_to_cpu_64(hdr->wqe_ptr) -
 572                                  sizeof(struct rte_mbuf));
 573                         /* Update dynamic field with userdata */
 574                         *rte_security_dynfield(inner) =
 575                                 (uint64_t)inb_priv->userdata;
 576
 577                         /* CPT result(struct cpt_cn10k_res_s) is at
 578                          * after first IOVA in meta
 579                          */
 580                         sg = (uint64_t *)(inner + 1);
 581                         res_w1 = sg[10];
 582
 583                         /* Clear checksum flags and update security flag */
 584                         *ol_flags &= ~(RTE_MBUF_F_RX_L4_CKSUM_MASK |
 585                                        RTE_MBUF_F_RX_IP_CKSUM_MASK);
 586                         *ol_flags |=
 587                                 (((res_w1 & 0xFF) == CPT_COMP_WARN) ?
 588                                  RTE_MBUF_F_RX_SEC_OFFLOAD :
 589                                  (RTE_MBUF_F_RX_SEC_OFFLOAD |
 590                                   RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED));
 591                         /* Calculate inner packet length */
 592                         len = ((res_w1 >> 16) & 0xFFFF) +
 593                                 hdr->w2.il3_off -
 594                                 sizeof(struct cpt_parse_hdr_s) -
 595                                 (w0 & 0x7);
 596                         /* Update pkt_len and data_len */
 597                         *rx_desc_field1 =
 598                                 vsetq_lane_u16(len, *rx_desc_field1, 2);
 599                         *rx_desc_field1 =
 600                                 vsetq_lane_u16(len, *rx_desc_field1, 4);
 601
 602                 } else if (!(hdr->w0.err_sum) && !(hdr->w0.reas_sts)) {
 603                         /* Reassembly success */
 604                         inner = nix_sec_reassemble_frags(hdr, cq_w1, cq_w5,
 605                                                          mbuf_init);
 606                         sg = (uint64_t *)(inner + 1);
 607                         res_w1 = sg[10];
 608
 609                         /* Update dynamic field with userdata */
 610                         *rte_security_dynfield(inner) =
 611                                 (uint64_t)inb_priv->userdata;
 612
 613                         /* Assume success */
 614                         *ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD;
 615
 616                         /* Update pkt_len and data_len */
 617                         *rx_desc_field1 = vsetq_lane_u16(inner->pkt_len,
 618                                                          *rx_desc_field1, 2);
 619                         *rx_desc_field1 = vsetq_lane_u16(inner->data_len,
 620                                                          *rx_desc_field1, 4);
 621
 622                         /* Data offset might be updated */
 623                         mbuf_init = *(uint64_t *)(&inner->rearm_data);
 624                         *rearm = vsetq_lane_u64(mbuf_init, *rearm, 0);
 625                 } else {
 626                         /* Reassembly failure */
 627                         inner = nix_sec_attach_frags(hdr, inb_priv, mbuf_init);
 628                         *ol_flags |= inner->ol_flags;
 629
 630                         /* Update pkt_len and data_len */
 631                         *rx_desc_field1 = vsetq_lane_u16(inner->pkt_len,
 632                                                          *rx_desc_field1, 2);
 633                         *rx_desc_field1 = vsetq_lane_u16(inner->data_len,
 634                                                          *rx_desc_field1, 4);
 635                 }
 636
 637                 /* Store meta in lmtline to free
 638                  * Assume all meta's from same aura.
 639                  */
 640                 *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf;
 641                 *loff = *loff + 1;
 642
 643                 /* Return inner mbuf */
 644                 return inner;
 645
 646         } else if (cq_w1 & BIT(11)) {
 647                 inner = (struct rte_mbuf *)(rte_be_to_cpu_64(hdr->wqe_ptr) -
 648                                             sizeof(struct rte_mbuf));
 649                 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */
 650                 w0 = hdr->w0.u64;
 651                 sa_idx = w0 >> 32;
 652
 653                 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx);
 654                 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa);
 655
 656                 /* Update dynamic field with userdata */
 657                 *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata;
 658
 659                 /* CPT result(struct cpt_cn10k_res_s) is at
 660                  * after first IOVA in meta
 661                  */
 662                 sg = (uint64_t *)(inner + 1);
 663                 res_w1 = sg[10];
 664
 665                 /* Clear checksum flags and update security flag */
 666                 *ol_flags &= ~(RTE_MBUF_F_RX_L4_CKSUM_MASK | RTE_MBUF_F_RX_IP_CKSUM_MASK);
 667                 *ol_flags |= (((res_w1 & 0xFF) == CPT_COMP_WARN) ?
 668                               RTE_MBUF_F_RX_SEC_OFFLOAD :
 669                               (RTE_MBUF_F_RX_SEC_OFFLOAD | RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED));
 670                 /* Calculate inner packet length */
 671                 len = ((res_w1 >> 16) & 0xFFFF) + hdr->w2.il3_off -
 672                         sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7);
 673                 /* Update pkt_len and data_len */
 674                 *rx_desc_field1 = vsetq_lane_u16(len, *rx_desc_field1, 2);
 675                 *rx_desc_field1 = vsetq_lane_u16(len, *rx_desc_field1, 4);
 676
 677                 /* Store meta in lmtline to free
 678                  * Assume all meta's from same aura.
 679                  */
 680                 *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf;
 681                 *loff = *loff + 1;
 682
 683                 /* Mark meta mbuf as put */
 684                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 0);
 685
 686                 /* Mark inner mbuf as get */
 687                 RTE_MEMPOOL_CHECK_COOKIES(inner->pool, (void **)&inner, 1, 1);
 688
 689                 /* Return inner mbuf */
 690                 return inner;
 691         }
 692
 693         /* Return same mbuf as it is not a decrypted pkt */
 694         return mbuf;
 695 }
 696 #endif
 697
 698 static __rte_always_inline uint32_t
 699 nix_ptype_get(const void *const lookup_mem, const uint64_t in)
 700 {
 701         const uint16_t *const ptype = lookup_mem;
 702         const uint16_t lh_lg_lf = (in & 0xFFF0000000000000) >> 52;
 703         const uint16_t tu_l2 = ptype[(in & 0x000FFFF000000000) >> 36];
 704         const uint16_t il4_tu = ptype[PTYPE_NON_TUNNEL_ARRAY_SZ + lh_lg_lf];
 705
 706         return (il4_tu << PTYPE_NON_TUNNEL_WIDTH) | tu_l2;
 707 }
 708
 709 static __rte_always_inline uint32_t
 710 nix_rx_olflags_get(const void *const lookup_mem, const uint64_t in)
 711 {
 712         const uint32_t *const ol_flags =
 713                 (const uint32_t *)((const uint8_t *)lookup_mem +
 714                                    PTYPE_ARRAY_SZ);
 715
 716         return ol_flags[(in & 0xfff00000) >> 20];
 717 }
 718
 719 static inline uint64_t
 720 nix_update_match_id(const uint16_t match_id, uint64_t ol_flags,
 721                     struct rte_mbuf *mbuf)
 722 {
 723         /* There is no separate bit to check match_id
 724          * is valid or not? and no flag to identify it is an
 725          * RTE_FLOW_ACTION_TYPE_FLAG vs RTE_FLOW_ACTION_TYPE_MARK
 726          * action. The former case addressed through 0 being invalid
 727          * value and inc/dec match_id pair when MARK is activated.
 728          * The later case addressed through defining
 729          * CNXK_FLOW_MARK_DEFAULT as value for
 730          * RTE_FLOW_ACTION_TYPE_MARK.
 731          * This would translate to not use
 732          * CNXK_FLOW_ACTION_FLAG_DEFAULT - 1 and
 733          * CNXK_FLOW_ACTION_FLAG_DEFAULT for match_id.
 734          * i.e valid mark_id's are from
 735          * 0 to CNXK_FLOW_ACTION_FLAG_DEFAULT - 2
 736          */
 737         if (likely(match_id)) {
 738                 ol_flags |= RTE_MBUF_F_RX_FDIR;
 739                 if (match_id != CNXK_FLOW_ACTION_FLAG_DEFAULT) {
 740                         ol_flags |= RTE_MBUF_F_RX_FDIR_ID;
 741                         mbuf->hash.fdir.hi = match_id - 1;
 742                 }
 743         }
 744
 745         return ol_flags;
 746 }
 747
 748 static __rte_always_inline void
 749 nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf,
 750                     uint64_t rearm, const uint16_t flags)
 751 {
 752         const rte_iova_t *iova_list;
 753         struct rte_mbuf *head;
 754         const rte_iova_t *eol;
 755         uint8_t nb_segs;
 756         uint64_t sg;
 757
 758         sg = *(const uint64_t *)(rx + 1);
 759         nb_segs = (sg >> 48) & 0x3;
 760
 761         if (nb_segs == 1 && !(flags & NIX_RX_SEC_REASSEMBLY_F)) {
 762                 mbuf->next = NULL;
 763                 return;
 764         }
 765
 766         mbuf->pkt_len = (rx->pkt_lenm1 + 1) - (flags & NIX_RX_OFFLOAD_TSTAMP_F ?
 767                                                CNXK_NIX_TIMESYNC_RX_OFFSET : 0);
 768         mbuf->data_len = (sg & 0xFFFF) - (flags & NIX_RX_OFFLOAD_TSTAMP_F ?
 769                                           CNXK_NIX_TIMESYNC_RX_OFFSET : 0);
 770         mbuf->nb_segs = nb_segs;
 771         sg = sg >> 16;
 772
 773         eol = ((const rte_iova_t *)(rx + 1) + ((rx->desc_sizem1 + 1) << 1));
 774         /* Skip SG_S and first IOVA*/
 775         iova_list = ((const rte_iova_t *)(rx + 1)) + 2;
 776         nb_segs--;
 777
 778         rearm = rearm & ~0xFFFF;
 779
 780         head = mbuf;
 781         while (nb_segs) {
 782                 mbuf->next = ((struct rte_mbuf *)*iova_list) - 1;
 783                 mbuf = mbuf->next;
 784
 785                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
 786
 787                 mbuf->data_len = sg & 0xFFFF;
 788                 sg = sg >> 16;
 789                 *(uint64_t *)(&mbuf->rearm_data) = rearm;
 790                 nb_segs--;
 791                 iova_list++;
 792
 793                 if (!nb_segs && (iova_list + 1 < eol)) {
 794                         sg = *(const uint64_t *)(iova_list);
 795                         nb_segs = (sg >> 48) & 0x3;
 796                         head->nb_segs += nb_segs;
 797                         iova_list = (const rte_iova_t *)(iova_list + 1);
 798                 }
 799         }
 800         mbuf->next = NULL;
 801 }
 802
 803 static __rte_always_inline void
 804 cn10k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag,
 805                       struct rte_mbuf *mbuf, const void *lookup_mem,
 806                       const uint64_t val, const uint16_t flag)
 807 {
 808         const union nix_rx_parse_u *rx =
 809                 (const union nix_rx_parse_u *)((const uint64_t *)cq + 1);
 810         const uint64_t w1 = *(const uint64_t *)rx;
 811         uint16_t len = rx->pkt_lenm1 + 1;
 812         uint64_t ol_flags = 0;
 813
 814         if (flag & NIX_RX_OFFLOAD_PTYPE_F)
 815                 mbuf->packet_type = nix_ptype_get(lookup_mem, w1);
 816         else
 817                 mbuf->packet_type = 0;
 818
 819         if (flag & NIX_RX_OFFLOAD_RSS_F) {
 820                 mbuf->hash.rss = tag;
 821                 ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 822         }
 823
 824         /* Skip rx ol flags extraction for Security packets */
 825         if ((!(flag & NIX_RX_SEC_REASSEMBLY_F) || !(w1 & BIT(11))) &&
 826                         flag & NIX_RX_OFFLOAD_CHECKSUM_F)
 827                 ol_flags |= nix_rx_olflags_get(lookup_mem, w1);
 828
 829         if (flag & NIX_RX_OFFLOAD_VLAN_STRIP_F) {
 830                 if (rx->vtag0_gone) {
 831                         ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
 832                         mbuf->vlan_tci = rx->vtag0_tci;
 833                 }
 834                 if (rx->vtag1_gone) {
 835                         ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED;
 836                         mbuf->vlan_tci_outer = rx->vtag1_tci;
 837                 }
 838         }
 839
 840         if (flag & NIX_RX_OFFLOAD_MARK_UPDATE_F)
 841                 ol_flags = nix_update_match_id(rx->match_id, ol_flags, mbuf);
 842
 843         /* Packet data length and ol flags is already updated for sec */
 844         if (flag & NIX_RX_SEC_REASSEMBLY_F && w1 & BIT_ULL(11)) {
 845                 mbuf->ol_flags |= ol_flags;
 846         } else {
 847                 mbuf->ol_flags = ol_flags;
 848                 mbuf->pkt_len = len;
 849                 mbuf->data_len = len;
 850                 *(uint64_t *)(&mbuf->rearm_data) = val;
 851         }
 852
 853         if (flag & NIX_RX_MULTI_SEG_F)
 854                 /*
 855                  * For multi segment packets, mbuf length correction according
 856                  * to Rx timestamp length will be handled later during
 857                  * timestamp data process.
 858                  * Hence, flag argument is not required.
 859                  */
 860                 nix_cqe_xtract_mseg(rx, mbuf, val, 0);
 861 }
 862
 863 static inline uint16_t
 864 nix_rx_nb_pkts(struct cn10k_eth_rxq *rxq, const uint64_t wdata,
 865                const uint16_t pkts, const uint32_t qmask)
 866 {
 867         uint32_t available = rxq->available;
 868
 869         /* Update the available count if cached value is not enough */
 870         if (unlikely(available < pkts)) {
 871                 uint64_t reg, head, tail;
 872
 873                 /* Use LDADDA version to avoid reorder */
 874                 reg = roc_atomic64_add_sync(wdata, rxq->cq_status);
 875                 /* CQ_OP_STATUS operation error */
 876                 if (reg & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) ||
 877                     reg & BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR))
 878                         return 0;
 879
 880                 tail = reg & 0xFFFFF;
 881                 head = (reg >> 20) & 0xFFFFF;
 882                 if (tail < head)
 883                         available = tail - head + qmask + 1;
 884                 else
 885                         available = tail - head;
 886
 887                 rxq->available = available;
 888         }
 889
 890         return RTE_MIN(pkts, available);
 891 }
 892
 893 static __rte_always_inline void
 894 cn10k_nix_mbuf_to_tstamp(struct rte_mbuf *mbuf,
 895                         struct cnxk_timesync_info *tstamp,
 896                         const uint8_t ts_enable, uint64_t *tstamp_ptr)
 897 {
 898         if (ts_enable) {
 899                 mbuf->pkt_len -= CNXK_NIX_TIMESYNC_RX_OFFSET;
 900                 mbuf->data_len -= CNXK_NIX_TIMESYNC_RX_OFFSET;
 901
 902                 /* Reading the rx timestamp inserted by CGX, viz at
 903                  * starting of the packet data.
 904                  */
 905                 *tstamp_ptr = ((*tstamp_ptr >> 32) * NSEC_PER_SEC) +
 906                         (*tstamp_ptr & 0xFFFFFFFFUL);
 907                 *cnxk_nix_timestamp_dynfield(mbuf, tstamp) =
 908                         rte_be_to_cpu_64(*tstamp_ptr);
 909                 /* RTE_MBUF_F_RX_IEEE1588_TMST flag needs to be set only in case
 910                  * PTP packets are received.
 911                  */
 912                 if (mbuf->packet_type == RTE_PTYPE_L2_ETHER_TIMESYNC) {
 913                         tstamp->rx_tstamp =
 914                                 *cnxk_nix_timestamp_dynfield(mbuf, tstamp);
 915                         tstamp->rx_ready = 1;
 916                         mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP |
 917                                 RTE_MBUF_F_RX_IEEE1588_TMST |
 918                                 tstamp->rx_tstamp_dynflag;
 919                 }
 920         }
 921 }
 922
 923 static __rte_always_inline uint16_t
 924 cn10k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts,
 925                     const uint16_t flags)
 926 {
 927         struct cn10k_eth_rxq *rxq = rx_queue;
 928         const uint64_t mbuf_init = rxq->mbuf_initializer;
 929         const void *lookup_mem = rxq->lookup_mem;
 930         const uint64_t data_off = rxq->data_off;
 931         const uintptr_t desc = rxq->desc;
 932         const uint64_t wdata = rxq->wdata;
 933         const uint32_t qmask = rxq->qmask;
 934         uint64_t lbase = rxq->lmt_base;
 935         uint16_t packets = 0, nb_pkts;
 936         uint8_t loff = 0, lnum = 0;
 937         uint32_t head = rxq->head;
 938         struct nix_cqe_hdr_s *cq;
 939         struct rte_mbuf *mbuf;
 940         uint64_t aura_handle;
 941         uint64_t sa_base;
 942         uint16_t lmt_id;
 943         uint64_t laddr;
 944
 945         nb_pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask);
 946
 947         if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 948                 aura_handle = rxq->aura_handle;
 949                 sa_base = rxq->sa_base;
 950                 sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1);
 951                 ROC_LMT_BASE_ID_GET(lbase, lmt_id);
 952                 laddr = lbase;
 953                 laddr += 8;
 954         }
 955
 956         while (packets < nb_pkts) {
 957                 /* Prefetch N desc ahead */
 958                 rte_prefetch_non_temporal(
 959                         (void *)(desc + (CQE_SZ((head + 2) & qmask))));
 960                 cq = (struct nix_cqe_hdr_s *)(desc + CQE_SZ(head));
 961
 962                 mbuf = nix_get_mbuf_from_cqe(cq, data_off);
 963
 964                 /* Mark mempool obj as "get" as it is alloc'ed by NIX */
 965                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
 966
 967                 /* Translate meta to mbuf */
 968                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 969                         const uint64_t cq_w1 = *((const uint64_t *)cq + 1);
 970                         const uint64_t cq_w5 = *((const uint64_t *)cq + 5);
 971
 972                         mbuf = nix_sec_meta_to_mbuf_sc(cq_w1, cq_w5, sa_base, laddr,
 973                                                        &loff, mbuf, data_off,
 974                                                        flags, mbuf_init);
 975                 }
 976
 977                 cn10k_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init,
 978                                       flags);
 979                 cn10k_nix_mbuf_to_tstamp(mbuf, rxq->tstamp,
 980                                         (flags & NIX_RX_OFFLOAD_TSTAMP_F),
 981                                         (uint64_t *)((uint8_t *)mbuf
 982                                                                 + data_off));
 983                 rx_pkts[packets++] = mbuf;
 984                 roc_prefetch_store_keep(mbuf);
 985                 head++;
 986                 head &= qmask;
 987
 988                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 989                         /* Flush when we don't have space for 4 meta */
 990                         if ((15 - loff) < 1) {
 991                                 nix_sec_flush_meta(laddr, lmt_id + lnum, loff,
 992                                                    aura_handle);
 993                                 lnum++;
 994                                 lnum &= BIT_ULL(ROC_LMT_LINES_PER_CORE_LOG2) -
 995                                         1;
 996                                 /* First pointer starts at 8B offset */
 997                                 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8);
 998                                 loff = 0;
 999                         }
1000                 }
1001         }
1002
1003         rxq->head = head;
1004         rxq->available -= nb_pkts;
1005
1006         /* Free all the CQs that we've processed */
1007         plt_write64((wdata | nb_pkts), rxq->cq_door);
1008
1009         /* Free remaining meta buffers if any */
1010         if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff)
1011                 nix_sec_flush_meta(laddr, lmt_id + lnum, loff, aura_handle);
1012
1013         if (flags & NIX_RX_OFFLOAD_SECURITY_F)
1014                 rte_io_wmb();
1015
1016         return nb_pkts;
1017 }
1018
1019 #if defined(RTE_ARCH_ARM64)
1020
1021 static __rte_always_inline uint64_t
1022 nix_vlan_update(const uint64_t w2, uint64_t ol_flags, uint8x16_t *f)
1023 {
1024         if (w2 & BIT_ULL(21) /* vtag0_gone */) {
1025                 ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
1026                 *f = vsetq_lane_u16((uint16_t)(w2 >> 32), *f, 5);
1027         }
1028
1029         return ol_flags;
1030 }
1031
1032 static __rte_always_inline uint64_t
1033 nix_qinq_update(const uint64_t w2, uint64_t ol_flags, struct rte_mbuf *mbuf)
1034 {
1035         if (w2 & BIT_ULL(23) /* vtag1_gone */) {
1036                 ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED;
1037                 mbuf->vlan_tci_outer = (uint16_t)(w2 >> 48);
1038         }
1039
1040         return ol_flags;
1041 }
1042
1043 static __rte_always_inline uint16_t
1044 cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts,
1045                            const uint16_t flags, void *lookup_mem,
1046                            struct cnxk_timesync_info *tstamp,
1047                            uintptr_t lmt_base)
1048 {
1049         struct cn10k_eth_rxq *rxq = args;
1050         const uint64_t mbuf_initializer = (flags & NIX_RX_VWQE_F) ?
1051                                                         *(uint64_t *)args :
1052                                                         rxq->mbuf_initializer;
1053         const uint64x2_t data_off = flags & NIX_RX_VWQE_F ?
1054                                         vdupq_n_u64(RTE_PKTMBUF_HEADROOM) :
1055                                         vdupq_n_u64(rxq->data_off);
1056         const uint32_t qmask = flags & NIX_RX_VWQE_F ? 0 : rxq->qmask;
1057         const uint64_t wdata = flags & NIX_RX_VWQE_F ? 0 : rxq->wdata;
1058         const uintptr_t desc = flags & NIX_RX_VWQE_F ? 0 : rxq->desc;
1059         uint64x2_t cq0_w8, cq1_w8, cq2_w8, cq3_w8, mbuf01, mbuf23;
1060         uint64_t ol_flags0, ol_flags1, ol_flags2, ol_flags3;
1061         uint64x2_t rearm0 = vdupq_n_u64(mbuf_initializer);
1062         uint64x2_t rearm1 = vdupq_n_u64(mbuf_initializer);
1063         uint64x2_t rearm2 = vdupq_n_u64(mbuf_initializer);
1064         uint64x2_t rearm3 = vdupq_n_u64(mbuf_initializer);
1065         struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3;
1066         uint64_t aura_handle, lbase, laddr;
1067         uint8_t loff = 0, lnum = 0, shft = 0;
1068         uint8x16_t f0, f1, f2, f3;
1069         uint16_t lmt_id, d_off;
1070         uint16_t packets = 0;
1071         uint16_t pkts_left;
1072         uintptr_t sa_base;
1073         uint32_t head;
1074         uintptr_t cq0;
1075
1076         if (!(flags & NIX_RX_VWQE_F)) {
1077                 lookup_mem = rxq->lookup_mem;
1078                 head = rxq->head;
1079
1080                 pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask);
1081                 pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1);
1082                 /* Packets has to be floor-aligned to NIX_DESCS_PER_LOOP */
1083                 pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
1084                 if (flags & NIX_RX_OFFLOAD_TSTAMP_F)
1085                         tstamp = rxq->tstamp;
1086         } else {
1087                 RTE_SET_USED(head);
1088         }
1089
1090         if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
1091                 if (flags & NIX_RX_VWQE_F) {
1092                         uint16_t port;
1093
1094                         mbuf0 = (struct rte_mbuf *)((uintptr_t)mbufs[0] -
1095                                                     sizeof(struct rte_mbuf));
1096                         /* Pick first mbuf's aura handle assuming all
1097                          * mbufs are from a vec and are from same RQ.
1098                          */
1099                         aura_handle = mbuf0->pool->pool_id;
1100                         /* Calculate offset from mbuf to actual data area */
1101                         d_off = ((uintptr_t)mbuf0->buf_addr - (uintptr_t)mbuf0);
1102                         d_off += (mbuf_initializer & 0xFFFF);
1103
1104                         /* Get SA Base from lookup tbl using port_id */
1105                         port = mbuf_initializer >> 48;
1106                         sa_base = cnxk_nix_sa_base_get(port, lookup_mem);
1107
1108                         lbase = lmt_base;
1109                 } else {
1110                         aura_handle = rxq->aura_handle;
1111                         d_off = rxq->data_off;
1112                         sa_base = rxq->sa_base;
1113                         lbase = rxq->lmt_base;
1114                 }
1115                 sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1);
1116                 ROC_LMT_BASE_ID_GET(lbase, lmt_id);
1117                 lnum = 0;
1118                 laddr = lbase;
1119                 laddr += 8;
1120         }
1121
1122         while (packets < pkts) {
1123                 if (!(flags & NIX_RX_VWQE_F)) {
1124                         /* Exit loop if head is about to wrap and become
1125                          * unaligned.
1126                          */
1127                         if (((head + NIX_DESCS_PER_LOOP - 1) & qmask) <
1128                             NIX_DESCS_PER_LOOP) {
1129                                 pkts_left += (pkts - packets);
1130                                 break;
1131                         }
1132
1133                         cq0 = desc + CQE_SZ(head);
1134                 } else {
1135                         cq0 = (uintptr_t)&mbufs[packets];
1136                 }
1137
1138                 if (flags & NIX_RX_VWQE_F) {
1139                         if (pkts - packets > 4) {
1140                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
1141                                         4, 0, flags));
1142                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
1143                                         5, 0, flags));
1144                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
1145                                         6, 0, flags));
1146                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
1147                                         7, 0, flags));
1148
1149                                 if (likely(pkts - packets > 8)) {
1150                                         rte_prefetch1(CQE_PTR_OFF(cq0,
1151                                                 8, 0, flags));
1152                                         rte_prefetch1(CQE_PTR_OFF(cq0,
1153                                                 9, 0, flags));
1154                                         rte_prefetch1(CQE_PTR_OFF(cq0,
1155                                                 10, 0, flags));
1156                                         rte_prefetch1(CQE_PTR_OFF(cq0,
1157                                                 11, 0, flags));
1158                                         if (pkts - packets > 12) {
1159                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
1160                                                         12, 0, flags));
1161                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
1162                                                         13, 0, flags));
1163                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
1164                                                         14, 0, flags));
1165                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
1166                                                         15, 0, flags));
1167                                         }
1168                                 }
1169
1170                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
1171                                         4, RTE_PKTMBUF_HEADROOM, flags));
1172                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
1173                                         5, RTE_PKTMBUF_HEADROOM, flags));
1174                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
1175                                         6, RTE_PKTMBUF_HEADROOM, flags));
1176                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
1177                                         7, RTE_PKTMBUF_HEADROOM, flags));
1178
1179                                 if (likely(pkts - packets > 8)) {
1180                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
1181                                                 8, RTE_PKTMBUF_HEADROOM, flags));
1182                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
1183                                                 9, RTE_PKTMBUF_HEADROOM, flags));
1184                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
1185                                                 10, RTE_PKTMBUF_HEADROOM, flags));
1186                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
1187                                                 11, RTE_PKTMBUF_HEADROOM, flags));
1188                                 }
1189                         }
1190                 } else {
1191                         if (pkts - packets > 4) {
1192                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 4, 64, flags));
1193                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 5, 64, flags));
1194                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 6, 64, flags));
1195                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 7, 64, flags));
1196                         }
1197                 }
1198
1199                 if (!(flags & NIX_RX_VWQE_F)) {
1200                         /* Get NIX_RX_SG_S for size and buffer pointer */
1201                         cq0_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 0, 64, flags));
1202                         cq1_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 1, 64, flags));
1203                         cq2_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 2, 64, flags));
1204                         cq3_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 3, 64, flags));
1205
1206                         /* Extract mbuf from NIX_RX_SG_S */
1207                         mbuf01 = vzip2q_u64(cq0_w8, cq1_w8);
1208                         mbuf23 = vzip2q_u64(cq2_w8, cq3_w8);
1209                         mbuf01 = vqsubq_u64(mbuf01, data_off);
1210                         mbuf23 = vqsubq_u64(mbuf23, data_off);
1211                 } else {
1212                         mbuf01 =
1213                                 vsubq_u64(vld1q_u64((uint64_t *)cq0), data_off);
1214                         mbuf23 = vsubq_u64(vld1q_u64((uint64_t *)(cq0 + 16)),
1215                                            data_off);
1216                 }
1217
1218                 /* Move mbufs to scalar registers for future use */
1219                 mbuf0 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 0);
1220                 mbuf1 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 1);
1221                 mbuf2 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 0);
1222                 mbuf3 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 1);
1223
1224                 if (!(flags & NIX_RX_VWQE_F)) {
1225                         /* Mask to get packet len from NIX_RX_SG_S */
1226                         const uint8x16_t shuf_msk = {
1227                                 0xFF, 0xFF, /* pkt_type set as unknown */
1228                                 0xFF, 0xFF, /* pkt_type set as unknown */
1229                                 0,    1,    /* octet 1~0, low 16 bits pkt_len */
1230                                 0xFF, 0xFF, /* skip high 16it pkt_len, zero out */
1231                                 0,    1,    /* octet 1~0, 16 bits data_len */
1232                                 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
1233
1234                         /* Form the rx_descriptor_fields1 with pkt_len and data_len */
1235                         f0 = vqtbl1q_u8(cq0_w8, shuf_msk);
1236                         f1 = vqtbl1q_u8(cq1_w8, shuf_msk);
1237                         f2 = vqtbl1q_u8(cq2_w8, shuf_msk);
1238                         f3 = vqtbl1q_u8(cq3_w8, shuf_msk);
1239                 }
1240                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
1241                         /* Prefetch probable CPT parse header area */
1242                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf0, d_off));
1243                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf1, d_off));
1244                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf2, d_off));
1245                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf3, d_off));
1246                 }
1247
1248                 /* Load CQE word0 and word 1 */
1249                 const uint64_t cq0_w0 = *CQE_PTR_OFF(cq0, 0, 0, flags);
1250                 const uint64_t cq0_w1 = *CQE_PTR_OFF(cq0, 0, 8, flags);
1251                 const uint64_t cq0_w2 = *CQE_PTR_OFF(cq0, 0, 16, flags);
1252                 const uint64_t cq1_w0 = *CQE_PTR_OFF(cq0, 1, 0, flags);
1253                 const uint64_t cq1_w1 = *CQE_PTR_OFF(cq0, 1, 8, flags);
1254                 const uint64_t cq1_w2 = *CQE_PTR_OFF(cq0, 1, 16, flags);
1255                 const uint64_t cq2_w0 = *CQE_PTR_OFF(cq0, 2, 0, flags);
1256                 const uint64_t cq2_w1 = *CQE_PTR_OFF(cq0, 2, 8, flags);
1257                 const uint64_t cq2_w2 = *CQE_PTR_OFF(cq0, 2, 16, flags);
1258                 const uint64_t cq3_w0 = *CQE_PTR_OFF(cq0, 3, 0, flags);
1259                 const uint64_t cq3_w1 = *CQE_PTR_OFF(cq0, 3, 8, flags);
1260                 const uint64_t cq3_w2 = *CQE_PTR_OFF(cq0, 3, 16, flags);
1261
1262                 if (flags & NIX_RX_VWQE_F) {
1263                         uint16_t psize0, psize1, psize2, psize3;
1264
1265                         psize0 = (cq0_w2 & 0xFFFF) + 1;
1266                         psize1 = (cq1_w2 & 0xFFFF) + 1;
1267                         psize2 = (cq2_w2 & 0xFFFF) + 1;
1268                         psize3 = (cq3_w2 & 0xFFFF) + 1;
1269
1270                         f0 = vdupq_n_u64(0);
1271                         f1 = vdupq_n_u64(0);
1272                         f2 = vdupq_n_u64(0);
1273                         f3 = vdupq_n_u64(0);
1274
1275                         f0 = vsetq_lane_u16(psize0, f0, 2);
1276                         f0 = vsetq_lane_u16(psize0, f0, 4);
1277
1278                         f1 = vsetq_lane_u16(psize1, f1, 2);
1279                         f1 = vsetq_lane_u16(psize1, f1, 4);
1280
1281                         f2 = vsetq_lane_u16(psize2, f2, 2);
1282                         f2 = vsetq_lane_u16(psize2, f2, 4);
1283
1284                         f3 = vsetq_lane_u16(psize3, f3, 2);
1285                         f3 = vsetq_lane_u16(psize3, f3, 4);
1286                 }
1287
1288                 if (flags & NIX_RX_OFFLOAD_RSS_F) {
1289                         /* Fill rss in the rx_descriptor_fields1 */
1290                         f0 = vsetq_lane_u32(cq0_w0, f0, 3);
1291                         f1 = vsetq_lane_u32(cq1_w0, f1, 3);
1292                         f2 = vsetq_lane_u32(cq2_w0, f2, 3);
1293                         f3 = vsetq_lane_u32(cq3_w0, f3, 3);
1294                         ol_flags0 = RTE_MBUF_F_RX_RSS_HASH;
1295                         ol_flags1 = RTE_MBUF_F_RX_RSS_HASH;
1296                         ol_flags2 = RTE_MBUF_F_RX_RSS_HASH;
1297                         ol_flags3 = RTE_MBUF_F_RX_RSS_HASH;
1298                 } else {
1299                         ol_flags0 = 0;
1300                         ol_flags1 = 0;
1301                         ol_flags2 = 0;
1302                         ol_flags3 = 0;
1303                 }
1304
1305                 if (flags & NIX_RX_OFFLOAD_PTYPE_F) {
1306                         /* Fill packet_type in the rx_descriptor_fields1 */
1307                         f0 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq0_w1),
1308                                             f0, 0);
1309                         f1 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq1_w1),
1310                                             f1, 0);
1311                         f2 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq2_w1),
1312                                             f2, 0);
1313                         f3 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq3_w1),
1314                                             f3, 0);
1315                 }
1316
1317                 if (flags & NIX_RX_OFFLOAD_CHECKSUM_F) {
1318                         ol_flags0 |= nix_rx_olflags_get(lookup_mem, cq0_w1);
1319                         ol_flags1 |= nix_rx_olflags_get(lookup_mem, cq1_w1);
1320                         ol_flags2 |= nix_rx_olflags_get(lookup_mem, cq2_w1);
1321                         ol_flags3 |= nix_rx_olflags_get(lookup_mem, cq3_w1);
1322                 }
1323
1324                 /* Mark mempool obj as "get" as it is alloc'ed by NIX */
1325                 RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1);
1326                 RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1);
1327                 RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1);
1328                 RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1);
1329
1330                 /* Translate meta to mbuf */
1331                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
1332                         uint64_t cq0_w5 = *(uint64_t *)(cq0 + CQE_SZ(0) + 40);
1333                         uint64_t cq1_w5 = *(uint64_t *)(cq0 + CQE_SZ(1) + 40);
1334                         uint64_t cq2_w5 = *(uint64_t *)(cq0 + CQE_SZ(2) + 40);
1335                         uint64_t cq3_w5 = *(uint64_t *)(cq0 + CQE_SZ(3) + 40);
1336
1337                         /* Initialize rearm data when reassembly is enabled as
1338                          * data offset might change.
1339                          */
1340                         if (flags & NIX_RX_REAS_F) {
1341                                 rearm0 = vdupq_n_u64(mbuf_initializer);
1342                                 rearm1 = vdupq_n_u64(mbuf_initializer);
1343                                 rearm2 = vdupq_n_u64(mbuf_initializer);
1344                                 rearm3 = vdupq_n_u64(mbuf_initializer);
1345                         }
1346
1347                         /* Checksum ol_flags will be cleared if mbuf is meta */
1348                         mbuf0 = nix_sec_meta_to_mbuf(cq0_w1, cq0_w5, sa_base, laddr,
1349                                                      &loff, mbuf0, d_off, &f0,
1350                                                      &ol_flags0, flags, &rearm0);
1351                         mbuf01 = vsetq_lane_u64((uint64_t)mbuf0, mbuf01, 0);
1352
1353                         mbuf1 = nix_sec_meta_to_mbuf(cq1_w1, cq1_w5, sa_base, laddr,
1354                                                      &loff, mbuf1, d_off, &f1,
1355                                                      &ol_flags1, flags, &rearm1);
1356                         mbuf01 = vsetq_lane_u64((uint64_t)mbuf1, mbuf01, 1);
1357
1358                         mbuf2 = nix_sec_meta_to_mbuf(cq2_w1, cq2_w5, sa_base, laddr,
1359                                                      &loff, mbuf2, d_off, &f2,
1360                                                      &ol_flags2, flags, &rearm2);
1361                         mbuf23 = vsetq_lane_u64((uint64_t)mbuf2, mbuf23, 0);
1362
1363                         mbuf3 = nix_sec_meta_to_mbuf(cq3_w1, cq3_w5, sa_base, laddr,
1364                                                      &loff, mbuf3, d_off, &f3,
1365                                                      &ol_flags3, flags, &rearm3);
1366                         mbuf23 = vsetq_lane_u64((uint64_t)mbuf3, mbuf23, 1);
1367                 }
1368
1369                 if (flags & NIX_RX_OFFLOAD_VLAN_STRIP_F) {
1370
1371                         ol_flags0 = nix_vlan_update(cq0_w2, ol_flags0, &f0);
1372                         ol_flags1 = nix_vlan_update(cq1_w2, ol_flags1, &f1);
1373                         ol_flags2 = nix_vlan_update(cq2_w2, ol_flags2, &f2);
1374                         ol_flags3 = nix_vlan_update(cq3_w2, ol_flags3, &f3);
1375
1376                         ol_flags0 = nix_qinq_update(cq0_w2, ol_flags0, mbuf0);
1377                         ol_flags1 = nix_qinq_update(cq1_w2, ol_flags1, mbuf1);
1378                         ol_flags2 = nix_qinq_update(cq2_w2, ol_flags2, mbuf2);
1379                         ol_flags3 = nix_qinq_update(cq3_w2, ol_flags3, mbuf3);
1380                 }
1381
1382                 if (flags & NIX_RX_OFFLOAD_MARK_UPDATE_F) {
1383                         ol_flags0 = nix_update_match_id(
1384                                 *(uint16_t *)CQE_PTR_OFF(cq0, 0, 38, flags),
1385                                 ol_flags0, mbuf0);
1386                         ol_flags1 = nix_update_match_id(
1387                                 *(uint16_t *)CQE_PTR_OFF(cq0, 1, 38, flags),
1388                                 ol_flags1, mbuf1);
1389                         ol_flags2 = nix_update_match_id(
1390                                 *(uint16_t *)CQE_PTR_OFF(cq0, 2, 38, flags),
1391                                 ol_flags2, mbuf2);
1392                         ol_flags3 = nix_update_match_id(
1393                                 *(uint16_t *)CQE_PTR_OFF(cq0, 3, 38, flags),
1394                                 ol_flags3, mbuf3);
1395                 }
1396
1397                 if (flags & NIX_RX_OFFLOAD_TSTAMP_F) {
1398                         const uint16x8_t len_off = {
1399                                 0,                           /* ptype   0:15 */
1400                                 0,                           /* ptype  16:32 */
1401                                 CNXK_NIX_TIMESYNC_RX_OFFSET, /* pktlen  0:15*/
1402                                 0,                           /* pktlen 16:32 */
1403                                 CNXK_NIX_TIMESYNC_RX_OFFSET, /* datalen 0:15 */
1404                                 0,
1405                                 0,
1406                                 0};
1407                         const uint32x4_t ptype = {RTE_PTYPE_L2_ETHER_TIMESYNC,
1408                                                   RTE_PTYPE_L2_ETHER_TIMESYNC,
1409                                                   RTE_PTYPE_L2_ETHER_TIMESYNC,
1410                                                   RTE_PTYPE_L2_ETHER_TIMESYNC};
1411                         const uint64_t ts_olf = RTE_MBUF_F_RX_IEEE1588_PTP |
1412                                                 RTE_MBUF_F_RX_IEEE1588_TMST |
1413                                                 tstamp->rx_tstamp_dynflag;
1414                         const uint32x4_t and_mask = {0x1, 0x2, 0x4, 0x8};
1415                         uint64x2_t ts01, ts23, mask;
1416                         uint64_t ts[4];
1417                         uint8_t res;
1418
1419                         /* Subtract timesync length from total pkt length. */
1420                         f0 = vsubq_u16(f0, len_off);
1421                         f1 = vsubq_u16(f1, len_off);
1422                         f2 = vsubq_u16(f2, len_off);
1423                         f3 = vsubq_u16(f3, len_off);
1424
1425                         /* Get the address of actual timestamp. */
1426                         ts01 = vaddq_u64(mbuf01, data_off);
1427                         ts23 = vaddq_u64(mbuf23, data_off);
1428                         /* Load timestamp from address. */
1429                         ts01 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts01,
1430                                                                           0),
1431                                               ts01, 0);
1432                         ts01 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts01,
1433                                                                           1),
1434                                               ts01, 1);
1435                         ts23 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts23,
1436                                                                           0),
1437                                               ts23, 0);
1438                         ts23 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts23,
1439                                                                           1),
1440                                               ts23, 1);
1441                         /* Convert from be to cpu byteorder. */
1442                         ts01 = vrev64q_u8(ts01);
1443                         ts23 = vrev64q_u8(ts23);
1444                         /* Store timestamp into scalar for later use. */
1445                         ts[0] = vgetq_lane_u64(ts01, 0);
1446                         ts[1] = vgetq_lane_u64(ts01, 1);
1447                         ts[2] = vgetq_lane_u64(ts23, 0);
1448                         ts[3] = vgetq_lane_u64(ts23, 1);
1449
1450                         /* Store timestamp into dynfield. */
1451                         *cnxk_nix_timestamp_dynfield(mbuf0, tstamp) = ts[0];
1452                         *cnxk_nix_timestamp_dynfield(mbuf1, tstamp) = ts[1];
1453                         *cnxk_nix_timestamp_dynfield(mbuf2, tstamp) = ts[2];
1454                         *cnxk_nix_timestamp_dynfield(mbuf3, tstamp) = ts[3];
1455
1456                         /* Generate ptype mask to filter L2 ether timesync */
1457                         mask = vdupq_n_u32(vgetq_lane_u32(f0, 0));
1458                         mask = vsetq_lane_u32(vgetq_lane_u32(f1, 0), mask, 1);
1459                         mask = vsetq_lane_u32(vgetq_lane_u32(f2, 0), mask, 2);
1460                         mask = vsetq_lane_u32(vgetq_lane_u32(f3, 0), mask, 3);
1461
1462                         /* Match against L2 ether timesync. */
1463                         mask = vceqq_u32(mask, ptype);
1464                         /* Convert from vector from scalar mask */
1465                         res = vaddvq_u32(vandq_u32(mask, and_mask));
1466                         res &= 0xF;
1467
1468                         if (res) {
1469                                 /* Fill in the ol_flags for any packets that
1470                                  * matched.
1471                                  */
1472                                 ol_flags0 |= ((res & 0x1) ? ts_olf : 0);
1473                                 ol_flags1 |= ((res & 0x2) ? ts_olf : 0);
1474                                 ol_flags2 |= ((res & 0x4) ? ts_olf : 0);
1475                                 ol_flags3 |= ((res & 0x8) ? ts_olf : 0);
1476
1477                                 /* Update Rxq timestamp with the latest
1478                                  * timestamp.
1479                                  */
1480                                 tstamp->rx_ready = 1;
1481                                 tstamp->rx_tstamp = ts[31 - __builtin_clz(res)];
1482                         }
1483                 }
1484
1485                 /* Form rearm_data with ol_flags */
1486                 rearm0 = vsetq_lane_u64(ol_flags0, rearm0, 1);
1487                 rearm1 = vsetq_lane_u64(ol_flags1, rearm1, 1);
1488                 rearm2 = vsetq_lane_u64(ol_flags2, rearm2, 1);
1489                 rearm3 = vsetq_lane_u64(ol_flags3, rearm3, 1);
1490
1491                 /* Update rx_descriptor_fields1 */
1492                 vst1q_u64((uint64_t *)mbuf0->rx_descriptor_fields1, f0);
1493                 vst1q_u64((uint64_t *)mbuf1->rx_descriptor_fields1, f1);
1494                 vst1q_u64((uint64_t *)mbuf2->rx_descriptor_fields1, f2);
1495                 vst1q_u64((uint64_t *)mbuf3->rx_descriptor_fields1, f3);
1496
1497                 /* Update rearm_data */
1498                 vst1q_u64((uint64_t *)mbuf0->rearm_data, rearm0);
1499                 vst1q_u64((uint64_t *)mbuf1->rearm_data, rearm1);
1500                 vst1q_u64((uint64_t *)mbuf2->rearm_data, rearm2);
1501                 vst1q_u64((uint64_t *)mbuf3->rearm_data, rearm3);
1502
1503                 /* Store the mbufs to rx_pkts */
1504                 vst1q_u64((uint64_t *)&mbufs[packets], mbuf01);
1505                 vst1q_u64((uint64_t *)&mbufs[packets + 2], mbuf23);
1506
1507                 if (flags & NIX_RX_MULTI_SEG_F) {
1508                         /* Multi segment is enable build mseg list for
1509                          * individual mbufs in scalar mode.
1510                          */
1511                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1512                                             (CQE_PTR_OFF(cq0, 0, 8, flags)),
1513                                             mbuf0, mbuf_initializer, flags);
1514                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1515                                             (CQE_PTR_OFF(cq0, 1, 8, flags)),
1516                                             mbuf1, mbuf_initializer, flags);
1517                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1518                                             (CQE_PTR_OFF(cq0, 2, 8, flags)),
1519                                             mbuf2, mbuf_initializer, flags);
1520                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1521                                             (CQE_PTR_OFF(cq0, 3, 8, flags)),
1522                                             mbuf3, mbuf_initializer, flags);
1523                 }
1524
1525                 /* Mark mempool obj as "get" as it is alloc'ed by NIX */
1526                 RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1);
1527                 RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1);
1528                 RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1);
1529                 RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1);
1530
1531                 nix_mbuf_validate_next(mbuf0);
1532                 nix_mbuf_validate_next(mbuf1);
1533                 nix_mbuf_validate_next(mbuf2);
1534                 nix_mbuf_validate_next(mbuf3);
1535
1536                 packets += NIX_DESCS_PER_LOOP;
1537
1538                 if (!(flags & NIX_RX_VWQE_F)) {
1539                         /* Advance head pointer and packets */
1540                         head += NIX_DESCS_PER_LOOP;
1541                         head &= qmask;
1542                 }
1543
1544                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
1545                         /* Check if lmtline border is crossed and adjust lnum */
1546                         if (loff > 15) {
1547                                 /* Update aura handle */
1548                                 *(uint64_t *)(laddr - 8) =
1549                                         (((uint64_t)(15 & 0x1) << 32) |
1550                                     roc_npa_aura_handle_to_aura(aura_handle));
1551                                 loff = loff - 15;
1552                                 shft += 3;
1553
1554                                 lnum++;
1555                                 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8);
1556                                 /* Pick the pointer from 16th index and put it
1557                                  * at end of this new line.
1558                                  */
1559                                 *(uint64_t *)(laddr + (loff << 3) - 8) =
1560                                         *(uint64_t *)(laddr - 8);
1561                         }
1562
1563                         /* Flush it when we are in 16th line and might
1564                          * overflow it
1565                          */
1566                         if (lnum >= 15 && loff >= 12) {
1567                                 /* 16 LMT Line size m1 */
1568                                 uint64_t data = BIT_ULL(48) - 1;
1569
1570                                 /* Update aura handle */
1571                                 *(uint64_t *)(laddr - 8) =
1572                                         (((uint64_t)(loff & 0x1) << 32) |
1573                                     roc_npa_aura_handle_to_aura(aura_handle));
1574
1575                                 data = (data & ~(0x7UL << shft)) |
1576                                        (((uint64_t)loff >> 1) << shft);
1577
1578                                 /* Send up to 16 lmt lines of pointers */
1579                                 nix_sec_flush_meta_burst(lmt_id, data, lnum + 1,
1580                                                          aura_handle);
1581                                 rte_io_wmb();
1582                                 lnum = 0;
1583                                 loff = 0;
1584                                 shft = 0;
1585                                 /* First pointer starts at 8B offset */
1586                                 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8);
1587                         }
1588                 }
1589         }
1590
1591         if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff) {
1592                 /* 16 LMT Line size m1 */
1593                 uint64_t data = BIT_ULL(48) - 1;
1594
1595                 /* Update aura handle */
1596                 *(uint64_t *)(laddr - 8) =
1597                         (((uint64_t)(loff & 0x1) << 32) |
1598                          roc_npa_aura_handle_to_aura(aura_handle));
1599
1600                 data = (data & ~(0x7UL << shft)) |
1601                        (((uint64_t)loff >> 1) << shft);
1602
1603                 /* Send up to 16 lmt lines of pointers */
1604                 nix_sec_flush_meta_burst(lmt_id, data, lnum + 1, aura_handle);
1605                 if (flags & NIX_RX_VWQE_F)
1606                         plt_io_wmb();
1607         }
1608
1609         if (flags & NIX_RX_VWQE_F)
1610                 return packets;
1611
1612         rxq->head = head;
1613         rxq->available -= packets;
1614
1615         rte_io_wmb();
1616         /* Free all the CQs that we've processed */
1617         plt_write64((rxq->wdata | packets), rxq->cq_door);
1618
1619         if (unlikely(pkts_left))
1620                 packets += cn10k_nix_recv_pkts(args, &mbufs[packets], pkts_left,
1621                                                flags);
1622
1623         return packets;
1624 }
1625
1626 #else
1627
1628 static inline uint16_t
1629 cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts,
1630                            const uint16_t flags, void *lookup_mem,
1631                            struct cnxk_timesync_info *tstamp,
1632                            uintptr_t lmt_base)
1633 {
1634         RTE_SET_USED(args);
1635         RTE_SET_USED(mbufs);
1636         RTE_SET_USED(pkts);
1637         RTE_SET_USED(flags);
1638         RTE_SET_USED(lookup_mem);
1639         RTE_SET_USED(tstamp);
1640         RTE_SET_USED(lmt_base);
1641
1642         return 0;
1643 }
1644
1645 #endif
1646
1647
1648 #define RSS_F     NIX_RX_OFFLOAD_RSS_F
1649 #define PTYPE_F   NIX_RX_OFFLOAD_PTYPE_F
1650 #define CKSUM_F   NIX_RX_OFFLOAD_CHECKSUM_F
1651 #define MARK_F    NIX_RX_OFFLOAD_MARK_UPDATE_F
1652 #define TS_F      NIX_RX_OFFLOAD_TSTAMP_F
1653 #define RX_VLAN_F NIX_RX_OFFLOAD_VLAN_STRIP_F
1654 #define R_SEC_F   NIX_RX_OFFLOAD_SECURITY_F
1655
1656 /* [R_SEC_F] [RX_VLAN_F] [TS] [MARK] [CKSUM] [PTYPE] [RSS] */
1657 #define NIX_RX_FASTPATH_MODES_0_15                                             \
1658         R(no_offload, NIX_RX_OFFLOAD_NONE)                                     \
1659         R(rss, RSS_F)                                                          \
1660         R(ptype, PTYPE_F)                                                      \
1661         R(ptype_rss, PTYPE_F | RSS_F)                                          \
1662         R(cksum, CKSUM_F)                                                      \
1663         R(cksum_rss, CKSUM_F | RSS_F)                                          \
1664         R(cksum_ptype, CKSUM_F | PTYPE_F)                                      \
1665         R(cksum_ptype_rss, CKSUM_F | PTYPE_F | RSS_F)                          \
1666         R(mark, MARK_F)                                                        \
1667         R(mark_rss, MARK_F | RSS_F)                                            \
1668         R(mark_ptype, MARK_F | PTYPE_F)                                        \
1669         R(mark_ptype_rss, MARK_F | PTYPE_F | RSS_F)                            \
1670         R(mark_cksum, MARK_F | CKSUM_F)                                        \
1671         R(mark_cksum_rss, MARK_F | CKSUM_F | RSS_F)                            \
1672         R(mark_cksum_ptype, MARK_F | CKSUM_F | PTYPE_F)                        \
1673         R(mark_cksum_ptype_rss, MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1674
1675 #define NIX_RX_FASTPATH_MODES_16_31                                            \
1676         R(ts, TS_F)                                                            \
1677         R(ts_rss, TS_F | RSS_F)                                                \
1678         R(ts_ptype, TS_F | PTYPE_F)                                            \
1679         R(ts_ptype_rss, TS_F | PTYPE_F | RSS_F)                                \
1680         R(ts_cksum, TS_F | CKSUM_F)                                            \
1681         R(ts_cksum_rss, TS_F | CKSUM_F | RSS_F)                                \
1682         R(ts_cksum_ptype, TS_F | CKSUM_F | PTYPE_F)                            \
1683         R(ts_cksum_ptype_rss, TS_F | CKSUM_F | PTYPE_F | RSS_F)                \
1684         R(ts_mark, TS_F | MARK_F)                                              \
1685         R(ts_mark_rss, TS_F | MARK_F | RSS_F)                                  \
1686         R(ts_mark_ptype, TS_F | MARK_F | PTYPE_F)                              \
1687         R(ts_mark_ptype_rss, TS_F | MARK_F | PTYPE_F | RSS_F)                  \
1688         R(ts_mark_cksum, TS_F | MARK_F | CKSUM_F)                              \
1689         R(ts_mark_cksum_rss, TS_F | MARK_F | CKSUM_F | RSS_F)                  \
1690         R(ts_mark_cksum_ptype, TS_F | MARK_F | CKSUM_F | PTYPE_F)              \
1691         R(ts_mark_cksum_ptype_rss, TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1692
1693 #define NIX_RX_FASTPATH_MODES_32_47                                            \
1694         R(vlan, RX_VLAN_F)                                                     \
1695         R(vlan_rss, RX_VLAN_F | RSS_F)                                         \
1696         R(vlan_ptype, RX_VLAN_F | PTYPE_F)                                     \
1697         R(vlan_ptype_rss, RX_VLAN_F | PTYPE_F | RSS_F)                         \
1698         R(vlan_cksum, RX_VLAN_F | CKSUM_F)                                     \
1699         R(vlan_cksum_rss, RX_VLAN_F | CKSUM_F | RSS_F)                         \
1700         R(vlan_cksum_ptype, RX_VLAN_F | CKSUM_F | PTYPE_F)                     \
1701         R(vlan_cksum_ptype_rss, RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)         \
1702         R(vlan_mark, RX_VLAN_F | MARK_F)                                       \
1703         R(vlan_mark_rss, RX_VLAN_F | MARK_F | RSS_F)                           \
1704         R(vlan_mark_ptype, RX_VLAN_F | MARK_F | PTYPE_F)                       \
1705         R(vlan_mark_ptype_rss, RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)           \
1706         R(vlan_mark_cksum, RX_VLAN_F | MARK_F | CKSUM_F)                       \
1707         R(vlan_mark_cksum_rss, RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)           \
1708         R(vlan_mark_cksum_ptype, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)       \
1709         R(vlan_mark_cksum_ptype_rss,                                           \
1710           RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1711
1712 #define NIX_RX_FASTPATH_MODES_48_63                                            \
1713         R(vlan_ts, RX_VLAN_F | TS_F)                                           \
1714         R(vlan_ts_rss, RX_VLAN_F | TS_F | RSS_F)                               \
1715         R(vlan_ts_ptype, RX_VLAN_F | TS_F | PTYPE_F)                           \
1716         R(vlan_ts_ptype_rss, RX_VLAN_F | TS_F | PTYPE_F | RSS_F)               \
1717         R(vlan_ts_cksum, RX_VLAN_F | TS_F | CKSUM_F)                           \
1718         R(vlan_ts_cksum_rss, RX_VLAN_F | TS_F | CKSUM_F | RSS_F)               \
1719         R(vlan_ts_cksum_ptype, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)           \
1720         R(vlan_ts_cksum_ptype_rss,                                             \
1721           RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)                        \
1722         R(vlan_ts_mark, RX_VLAN_F | TS_F | MARK_F)                             \
1723         R(vlan_ts_mark_rss, RX_VLAN_F | TS_F | MARK_F | RSS_F)                 \
1724         R(vlan_ts_mark_ptype, RX_VLAN_F | TS_F | MARK_F | PTYPE_F)             \
1725         R(vlan_ts_mark_ptype_rss, RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
1726         R(vlan_ts_mark_cksum, RX_VLAN_F | TS_F | MARK_F | CKSUM_F)             \
1727         R(vlan_ts_mark_cksum_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
1728         R(vlan_ts_mark_cksum_ptype,                                            \
1729           RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                       \
1730         R(vlan_ts_mark_cksum_ptype_rss,                                        \
1731           RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1732
1733 #define NIX_RX_FASTPATH_MODES_64_79                                            \
1734         R(sec, R_SEC_F)                                                        \
1735         R(sec_rss, R_SEC_F | RSS_F)                                            \
1736         R(sec_ptype, R_SEC_F | PTYPE_F)                                        \
1737         R(sec_ptype_rss, R_SEC_F | PTYPE_F | RSS_F)                            \
1738         R(sec_cksum, R_SEC_F | CKSUM_F)                                        \
1739         R(sec_cksum_rss, R_SEC_F | CKSUM_F | RSS_F)                            \
1740         R(sec_cksum_ptype, R_SEC_F | CKSUM_F | PTYPE_F)                        \
1741         R(sec_cksum_ptype_rss, R_SEC_F | CKSUM_F | PTYPE_F | RSS_F)            \
1742         R(sec_mark, R_SEC_F | MARK_F)                                          \
1743         R(sec_mark_rss, R_SEC_F | MARK_F | RSS_F)                              \
1744         R(sec_mark_ptype, R_SEC_F | MARK_F | PTYPE_F)                          \
1745         R(sec_mark_ptype_rss, R_SEC_F | MARK_F | PTYPE_F | RSS_F)              \
1746         R(sec_mark_cksum, R_SEC_F | MARK_F | CKSUM_F)                          \
1747         R(sec_mark_cksum_rss, R_SEC_F | MARK_F | CKSUM_F | RSS_F)              \
1748         R(sec_mark_cksum_ptype, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F)          \
1749         R(sec_mark_cksum_ptype_rss,                                            \
1750           R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1751
1752 #define NIX_RX_FASTPATH_MODES_80_95                                            \
1753         R(sec_ts, R_SEC_F | TS_F)                                              \
1754         R(sec_ts_rss, R_SEC_F | TS_F | RSS_F)                                  \
1755         R(sec_ts_ptype, R_SEC_F | TS_F | PTYPE_F)                              \
1756         R(sec_ts_ptype_rss, R_SEC_F | TS_F | PTYPE_F | RSS_F)                  \
1757         R(sec_ts_cksum, R_SEC_F | TS_F | CKSUM_F)                              \
1758         R(sec_ts_cksum_rss, R_SEC_F | TS_F | CKSUM_F | RSS_F)                  \
1759         R(sec_ts_cksum_ptype, R_SEC_F | TS_F | CKSUM_F | PTYPE_F)              \
1760         R(sec_ts_cksum_ptype_rss, R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)  \
1761         R(sec_ts_mark, R_SEC_F | TS_F | MARK_F)                                \
1762         R(sec_ts_mark_rss, R_SEC_F | TS_F | MARK_F | RSS_F)                    \
1763         R(sec_ts_mark_ptype, R_SEC_F | TS_F | MARK_F | PTYPE_F)                \
1764         R(sec_ts_mark_ptype_rss, R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F)    \
1765         R(sec_ts_mark_cksum, R_SEC_F | TS_F | MARK_F | CKSUM_F)                \
1766         R(sec_ts_mark_cksum_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F)    \
1767         R(sec_ts_mark_cksum_ptype,                                             \
1768           R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                         \
1769         R(sec_ts_mark_cksum_ptype_rss,                                         \
1770           R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1771
1772 #define NIX_RX_FASTPATH_MODES_96_111                                           \
1773         R(sec_vlan, R_SEC_F | RX_VLAN_F)                                       \
1774         R(sec_vlan_rss, R_SEC_F | RX_VLAN_F | RSS_F)                           \
1775         R(sec_vlan_ptype, R_SEC_F | RX_VLAN_F | PTYPE_F)                       \
1776         R(sec_vlan_ptype_rss, R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F)           \
1777         R(sec_vlan_cksum, R_SEC_F | RX_VLAN_F | CKSUM_F)                       \
1778         R(sec_vlan_cksum_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F)           \
1779         R(sec_vlan_cksum_ptype, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F)       \
1780         R(sec_vlan_cksum_ptype_rss,                                            \
1781           R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)                     \
1782         R(sec_vlan_mark, R_SEC_F | RX_VLAN_F | MARK_F)                         \
1783         R(sec_vlan_mark_rss, R_SEC_F | RX_VLAN_F | MARK_F | RSS_F)             \
1784         R(sec_vlan_mark_ptype, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F)         \
1785         R(sec_vlan_mark_ptype_rss,                                             \
1786           R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)                      \
1787         R(sec_vlan_mark_cksum, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F)         \
1788         R(sec_vlan_mark_cksum_rss,                                             \
1789           R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)                      \
1790         R(sec_vlan_mark_cksum_ptype,                                           \
1791           R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)                    \
1792         R(sec_vlan_mark_cksum_ptype_rss,                                       \
1793           R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1794
1795 #define NIX_RX_FASTPATH_MODES_112_127                                          \
1796         R(sec_vlan_ts, R_SEC_F | RX_VLAN_F | TS_F)                             \
1797         R(sec_vlan_ts_rss, R_SEC_F | RX_VLAN_F | TS_F | RSS_F)                 \
1798         R(sec_vlan_ts_ptype, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F)             \
1799         R(sec_vlan_ts_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
1800         R(sec_vlan_ts_cksum, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F)             \
1801         R(sec_vlan_ts_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
1802         R(sec_vlan_ts_cksum_ptype,                                             \
1803           R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)                      \
1804         R(sec_vlan_ts_cksum_ptype_rss,                                         \
1805           R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)              \
1806         R(sec_vlan_ts_mark, R_SEC_F | RX_VLAN_F | TS_F | MARK_F)               \
1807         R(sec_vlan_ts_mark_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F)   \
1808         R(sec_vlan_ts_mark_ptype,                                              \
1809           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F)                       \
1810         R(sec_vlan_ts_mark_ptype_rss,                                          \
1811           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F)               \
1812         R(sec_vlan_ts_mark_cksum,                                              \
1813           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F)                       \
1814         R(sec_vlan_ts_mark_cksum_rss,                                          \
1815           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F)               \
1816         R(sec_vlan_ts_mark_cksum_ptype,                                        \
1817           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)             \
1818         R(sec_vlan_ts_mark_cksum_ptype_rss,                                    \
1819           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1820
1821
1822 #define NIX_RX_FASTPATH_MODES                                                  \
1823         NIX_RX_FASTPATH_MODES_0_15                                             \
1824         NIX_RX_FASTPATH_MODES_16_31                                            \
1825         NIX_RX_FASTPATH_MODES_32_47                                            \
1826         NIX_RX_FASTPATH_MODES_48_63                                            \
1827         NIX_RX_FASTPATH_MODES_64_79                                            \
1828         NIX_RX_FASTPATH_MODES_80_95                                            \
1829         NIX_RX_FASTPATH_MODES_96_111                                           \
1830         NIX_RX_FASTPATH_MODES_112_127                                          \
1831
1832 #define R(name, flags)                                                         \
1833         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_##name(          \
1834                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1835         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_mseg_##name(     \
1836                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1837         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_##name(      \
1838                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1839         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_mseg_##name( \
1840                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1841         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_##name(     \
1842                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1843         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_mseg_##name(\
1844                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1845         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_vec_##name( \
1846                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1847         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_vec_mseg_##name( \
1848                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);
1849
1850 NIX_RX_FASTPATH_MODES
1851 #undef R
1852
1853 #define NIX_RX_RECV(fn, flags)                                                 \
1854         uint16_t __rte_noinline __rte_hot fn(                                  \
1855                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
1856         {                                                                      \
1857                 return cn10k_nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags));  \
1858         }
1859
1860 #define NIX_RX_RECV_MSEG(fn, flags) NIX_RX_RECV(fn, flags | NIX_RX_MULTI_SEG_F)
1861
1862 #define NIX_RX_RECV_VEC(fn, flags)                                             \
1863         uint16_t __rte_noinline __rte_hot fn(                                  \
1864                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
1865         {                                                                      \
1866                 return cn10k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts,     \
1867                                                   (flags), NULL, NULL, 0);     \
1868         }
1869
1870 #define NIX_RX_RECV_VEC_MSEG(fn, flags)                                        \
1871         NIX_RX_RECV_VEC(fn, flags | NIX_RX_MULTI_SEG_F)
1872
1873 #endif /* __CN10K_RX_H__ */