drivers/net/cnxk/cn10k_rx.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4 #ifndef __CN10K_RX_H__
   5 #define __CN10K_RX_H__
   6
   7 #include <rte_ether.h>
   8 #include <rte_vect.h>
   9
  10 #include <cnxk_ethdev.h>
  11
  12 #define NIX_RX_OFFLOAD_NONE          (0)
  13 #define NIX_RX_OFFLOAD_RSS_F         BIT(0)
  14 #define NIX_RX_OFFLOAD_PTYPE_F       BIT(1)
  15 #define NIX_RX_OFFLOAD_CHECKSUM_F    BIT(2)
  16 #define NIX_RX_OFFLOAD_MARK_UPDATE_F BIT(3)
  17 #define NIX_RX_OFFLOAD_TSTAMP_F      BIT(4)
  18 #define NIX_RX_OFFLOAD_VLAN_STRIP_F  BIT(5)
  19 #define NIX_RX_OFFLOAD_SECURITY_F    BIT(6)
  20 #define NIX_RX_OFFLOAD_MAX           (NIX_RX_OFFLOAD_SECURITY_F << 1)
  21
  22 /* Flags to control cqe_to_mbuf conversion function.
  23  * Defining it from backwards to denote its been
  24  * not used as offload flags to pick function
  25  */
  26 #define NIX_RX_VWQE_F      BIT(13)
  27 #define NIX_RX_MULTI_SEG_F BIT(14)
  28 #define CPT_RX_WQE_F       BIT(15)
  29
  30 #define CNXK_NIX_CQ_ENTRY_SZ 128
  31 #define NIX_DESCS_PER_LOOP   4
  32 #define CQE_CAST(x)          ((struct nix_cqe_hdr_s *)(x))
  33 #define CQE_SZ(x)            ((x) * CNXK_NIX_CQ_ENTRY_SZ)
  34
  35 #define CQE_PTR_OFF(b, i, o, f)                                                \
  36         (((f) & NIX_RX_VWQE_F) ?                                               \
  37                        (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) + (o)) : \
  38                        (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) + (o)))
  39 #define CQE_PTR_DIFF(b, i, o, f)                                               \
  40         (((f) & NIX_RX_VWQE_F) ?                                               \
  41                  (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) - (o)) :       \
  42                        (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) - (o)))
  43
  44 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
  45 static inline void
  46 nix_mbuf_validate_next(struct rte_mbuf *m)
  47 {
  48         if (m->nb_segs == 1 && m->next) {
  49                 rte_panic("mbuf->next[%p] valid when mbuf->nb_segs is %d",
  50                         m->next, m->nb_segs);
  51         }
  52 }
  53 #else
  54 static inline void
  55 nix_mbuf_validate_next(struct rte_mbuf *m)
  56 {
  57         RTE_SET_USED(m);
  58 }
  59 #endif
  60
  61 union mbuf_initializer {
  62         struct {
  63                 uint16_t data_off;
  64                 uint16_t refcnt;
  65                 uint16_t nb_segs;
  66                 uint16_t port;
  67         } fields;
  68         uint64_t value;
  69 };
  70
  71 static __rte_always_inline uint64_t
  72 nix_clear_data_off(uint64_t oldval)
  73 {
  74         union mbuf_initializer mbuf_init = {.value = oldval};
  75
  76         mbuf_init.fields.data_off = 0;
  77         return mbuf_init.value;
  78 }
  79
  80 static __rte_always_inline struct rte_mbuf *
  81 nix_get_mbuf_from_cqe(void *cq, const uint64_t data_off)
  82 {
  83         rte_iova_t buff;
  84
  85         /* Skip CQE, NIX_RX_PARSE_S and SG HDR(9 DWORDs) and peek buff addr */
  86         buff = *((rte_iova_t *)((uint64_t *)cq + 9));
  87         return (struct rte_mbuf *)(buff - data_off);
  88 }
  89
  90 static __rte_always_inline void
  91 nix_sec_flush_meta_burst(uint16_t lmt_id, uint64_t data, uint16_t lnum,
  92                          uintptr_t aura_handle)
  93 {
  94         uint64_t pa;
  95
  96         /* Prepare PA and Data */
  97         pa = roc_npa_aura_handle_to_base(aura_handle) + NPA_LF_AURA_BATCH_FREE0;
  98         pa |= ((data & 0x7) << 4);
  99
 100         data >>= 3;
 101         data <<= 19;
 102         data |= (uint64_t)lmt_id;
 103         data |= (uint64_t)(lnum - 1) << 12;
 104
 105         roc_lmt_submit_steorl(data, pa);
 106 }
 107
 108 static __rte_always_inline void
 109 nix_sec_flush_meta(uintptr_t laddr, uint16_t lmt_id, uint8_t loff,
 110                    uintptr_t aura_handle)
 111 {
 112         uint64_t pa;
 113
 114         /* laddr is pointing to first pointer */
 115         laddr -= 8;
 116
 117         /* Trigger free either on lmtline full or different aura handle */
 118         pa = roc_npa_aura_handle_to_base(aura_handle) + NPA_LF_AURA_BATCH_FREE0;
 119
 120         /* Update aura handle */
 121         *(uint64_t *)laddr = (((uint64_t)(loff & 0x1) << 32) |
 122                               roc_npa_aura_handle_to_aura(aura_handle));
 123
 124         pa |= ((uint64_t)(loff >> 1) << 4);
 125         roc_lmt_submit_steorl(lmt_id, pa);
 126 }
 127
 128 static __rte_always_inline struct rte_mbuf *
 129 nix_sec_meta_to_mbuf_sc(uint64_t cq_w1, const uint64_t sa_base, uintptr_t laddr,
 130                         uint8_t *loff, struct rte_mbuf *mbuf, uint16_t data_off)
 131 {
 132         const void *__p = (void *)((uintptr_t)mbuf + (uint16_t)data_off);
 133         const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)__p;
 134         struct cn10k_inb_priv_data *inb_priv;
 135         struct rte_mbuf *inner;
 136         uint32_t sa_idx;
 137         void *inb_sa;
 138         uint64_t w0;
 139
 140         if (cq_w1 & BIT(11)) {
 141                 inner = (struct rte_mbuf *)(rte_be_to_cpu_64(hdr->wqe_ptr) -
 142                                             sizeof(struct rte_mbuf));
 143
 144                 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */
 145                 w0 = hdr->w0.u64;
 146                 sa_idx = w0 >> 32;
 147
 148                 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx);
 149                 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa);
 150
 151                 /* Update dynamic field with userdata */
 152                 *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata;
 153
 154                 /* Update l2 hdr length first */
 155                 inner->pkt_len = (hdr->w2.il3_off -
 156                                   sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7));
 157
 158                 /* Store meta in lmtline to free
 159                  * Assume all meta's from same aura.
 160                  */
 161                 *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf;
 162                 *loff = *loff + 1;
 163
 164                 /* Mark meta mbuf as put */
 165                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 0);
 166
 167                 /* Mark inner mbuf as get */
 168                 RTE_MEMPOOL_CHECK_COOKIES(inner->pool, (void **)&inner, 1, 1);
 169
 170                 return inner;
 171         }
 172         return mbuf;
 173 }
 174
 175 #if defined(RTE_ARCH_ARM64)
 176
 177 static __rte_always_inline struct rte_mbuf *
 178 nix_sec_meta_to_mbuf(uint64_t cq_w1, uintptr_t sa_base, uintptr_t laddr,
 179                      uint8_t *loff, struct rte_mbuf *mbuf, uint16_t data_off,
 180                      uint8x16_t *rx_desc_field1, uint64_t *ol_flags)
 181 {
 182         const void *__p = (void *)((uintptr_t)mbuf + (uint16_t)data_off);
 183         const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)__p;
 184         struct cn10k_inb_priv_data *inb_priv;
 185         struct rte_mbuf *inner;
 186         uint64_t *sg, res_w1;
 187         uint32_t sa_idx;
 188         void *inb_sa;
 189         uint16_t len;
 190         uint64_t w0;
 191
 192         if (cq_w1 & BIT(11)) {
 193                 inner = (struct rte_mbuf *)(rte_be_to_cpu_64(hdr->wqe_ptr) -
 194                                             sizeof(struct rte_mbuf));
 195                 /* Get SPI from CPT_PARSE_S's cookie(already swapped) */
 196                 w0 = hdr->w0.u64;
 197                 sa_idx = w0 >> 32;
 198
 199                 inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx);
 200                 inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa);
 201
 202                 /* Update dynamic field with userdata */
 203                 *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata;
 204
 205                 /* CPT result(struct cpt_cn10k_res_s) is at
 206                  * after first IOVA in meta
 207                  */
 208                 sg = (uint64_t *)(inner + 1);
 209                 res_w1 = sg[10];
 210
 211                 /* Clear checksum flags and update security flag */
 212                 *ol_flags &= ~(RTE_MBUF_F_RX_L4_CKSUM_MASK | RTE_MBUF_F_RX_IP_CKSUM_MASK);
 213                 *ol_flags |= (((res_w1 & 0xFF) == CPT_COMP_WARN) ?
 214                               RTE_MBUF_F_RX_SEC_OFFLOAD :
 215                               (RTE_MBUF_F_RX_SEC_OFFLOAD | RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED));
 216                 /* Calculate inner packet length */
 217                 len = ((res_w1 >> 16) & 0xFFFF) + hdr->w2.il3_off -
 218                         sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7);
 219                 /* Update pkt_len and data_len */
 220                 *rx_desc_field1 = vsetq_lane_u16(len, *rx_desc_field1, 2);
 221                 *rx_desc_field1 = vsetq_lane_u16(len, *rx_desc_field1, 4);
 222
 223                 /* Store meta in lmtline to free
 224                  * Assume all meta's from same aura.
 225                  */
 226                 *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf;
 227                 *loff = *loff + 1;
 228
 229                 /* Mark meta mbuf as put */
 230                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 0);
 231
 232                 /* Mark inner mbuf as get */
 233                 RTE_MEMPOOL_CHECK_COOKIES(inner->pool, (void **)&inner, 1, 1);
 234
 235                 /* Return inner mbuf */
 236                 return inner;
 237         }
 238
 239         /* Return same mbuf as it is not a decrypted pkt */
 240         return mbuf;
 241 }
 242 #endif
 243
 244 static __rte_always_inline uint32_t
 245 nix_ptype_get(const void *const lookup_mem, const uint64_t in)
 246 {
 247         const uint16_t *const ptype = lookup_mem;
 248         const uint16_t lh_lg_lf = (in & 0xFFF0000000000000) >> 52;
 249         const uint16_t tu_l2 = ptype[(in & 0x000FFFF000000000) >> 36];
 250         const uint16_t il4_tu = ptype[PTYPE_NON_TUNNEL_ARRAY_SZ + lh_lg_lf];
 251
 252         return (il4_tu << PTYPE_NON_TUNNEL_WIDTH) | tu_l2;
 253 }
 254
 255 static __rte_always_inline uint32_t
 256 nix_rx_olflags_get(const void *const lookup_mem, const uint64_t in)
 257 {
 258         const uint32_t *const ol_flags =
 259                 (const uint32_t *)((const uint8_t *)lookup_mem +
 260                                    PTYPE_ARRAY_SZ);
 261
 262         return ol_flags[(in & 0xfff00000) >> 20];
 263 }
 264
 265 static inline uint64_t
 266 nix_update_match_id(const uint16_t match_id, uint64_t ol_flags,
 267                     struct rte_mbuf *mbuf)
 268 {
 269         /* There is no separate bit to check match_id
 270          * is valid or not? and no flag to identify it is an
 271          * RTE_FLOW_ACTION_TYPE_FLAG vs RTE_FLOW_ACTION_TYPE_MARK
 272          * action. The former case addressed through 0 being invalid
 273          * value and inc/dec match_id pair when MARK is activated.
 274          * The later case addressed through defining
 275          * CNXK_FLOW_MARK_DEFAULT as value for
 276          * RTE_FLOW_ACTION_TYPE_MARK.
 277          * This would translate to not use
 278          * CNXK_FLOW_ACTION_FLAG_DEFAULT - 1 and
 279          * CNXK_FLOW_ACTION_FLAG_DEFAULT for match_id.
 280          * i.e valid mark_id's are from
 281          * 0 to CNXK_FLOW_ACTION_FLAG_DEFAULT - 2
 282          */
 283         if (likely(match_id)) {
 284                 ol_flags |= RTE_MBUF_F_RX_FDIR;
 285                 if (match_id != CNXK_FLOW_ACTION_FLAG_DEFAULT) {
 286                         ol_flags |= RTE_MBUF_F_RX_FDIR_ID;
 287                         mbuf->hash.fdir.hi = match_id - 1;
 288                 }
 289         }
 290
 291         return ol_flags;
 292 }
 293
 294 static __rte_always_inline void
 295 nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf,
 296                     uint64_t rearm, const uint16_t flags)
 297 {
 298         const rte_iova_t *iova_list;
 299         struct rte_mbuf *head;
 300         const rte_iova_t *eol;
 301         uint8_t nb_segs;
 302         uint64_t sg;
 303
 304         sg = *(const uint64_t *)(rx + 1);
 305         nb_segs = (sg >> 48) & 0x3;
 306
 307         if (nb_segs == 1) {
 308                 mbuf->next = NULL;
 309                 return;
 310         }
 311
 312         mbuf->pkt_len = (rx->pkt_lenm1 + 1) - (flags & NIX_RX_OFFLOAD_TSTAMP_F ?
 313                                                CNXK_NIX_TIMESYNC_RX_OFFSET : 0);
 314         mbuf->data_len = (sg & 0xFFFF) - (flags & NIX_RX_OFFLOAD_TSTAMP_F ?
 315                                           CNXK_NIX_TIMESYNC_RX_OFFSET : 0);
 316         mbuf->nb_segs = nb_segs;
 317         sg = sg >> 16;
 318
 319         eol = ((const rte_iova_t *)(rx + 1) + ((rx->desc_sizem1 + 1) << 1));
 320         /* Skip SG_S and first IOVA*/
 321         iova_list = ((const rte_iova_t *)(rx + 1)) + 2;
 322         nb_segs--;
 323
 324         rearm = rearm & ~0xFFFF;
 325
 326         head = mbuf;
 327         while (nb_segs) {
 328                 mbuf->next = ((struct rte_mbuf *)*iova_list) - 1;
 329                 mbuf = mbuf->next;
 330
 331                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
 332
 333                 mbuf->data_len = sg & 0xFFFF;
 334                 sg = sg >> 16;
 335                 *(uint64_t *)(&mbuf->rearm_data) = rearm;
 336                 nb_segs--;
 337                 iova_list++;
 338
 339                 if (!nb_segs && (iova_list + 1 < eol)) {
 340                         sg = *(const uint64_t *)(iova_list);
 341                         nb_segs = (sg >> 48) & 0x3;
 342                         head->nb_segs += nb_segs;
 343                         iova_list = (const rte_iova_t *)(iova_list + 1);
 344                 }
 345         }
 346         mbuf->next = NULL;
 347 }
 348
 349 static __rte_always_inline void
 350 cn10k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag,
 351                       struct rte_mbuf *mbuf, const void *lookup_mem,
 352                       const uint64_t val, const uint16_t flag)
 353 {
 354         const union nix_rx_parse_u *rx =
 355                 (const union nix_rx_parse_u *)((const uint64_t *)cq + 1);
 356         const uint64_t w1 = *(const uint64_t *)rx;
 357         uint16_t len = rx->pkt_lenm1 + 1;
 358         uint64_t ol_flags = 0;
 359
 360         if (flag & NIX_RX_OFFLOAD_PTYPE_F)
 361                 mbuf->packet_type = nix_ptype_get(lookup_mem, w1);
 362         else
 363                 mbuf->packet_type = 0;
 364
 365         if (flag & NIX_RX_OFFLOAD_RSS_F) {
 366                 mbuf->hash.rss = tag;
 367                 ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 368         }
 369
 370         /* Process Security packets */
 371         if (flag & NIX_RX_OFFLOAD_SECURITY_F) {
 372                 if (w1 & BIT(11)) {
 373                         /* CPT result(struct cpt_cn10k_res_s) is at
 374                          * after first IOVA in meta
 375                          */
 376                         const uint64_t *sg = (const uint64_t *)(mbuf + 1);
 377                         const uint64_t res_w1 = sg[10];
 378                         const uint16_t uc_cc = res_w1 & 0xFF;
 379
 380                         /* Rlen */
 381                         len = ((res_w1 >> 16) & 0xFFFF) + mbuf->pkt_len;
 382                         ol_flags |= ((uc_cc == CPT_COMP_WARN) ?
 383                                                    RTE_MBUF_F_RX_SEC_OFFLOAD :
 384                                                    (RTE_MBUF_F_RX_SEC_OFFLOAD |
 385                                               RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED));
 386                 } else {
 387                         if (flag & NIX_RX_OFFLOAD_CHECKSUM_F)
 388                                 ol_flags |= nix_rx_olflags_get(lookup_mem, w1);
 389                 }
 390         } else {
 391                 if (flag & NIX_RX_OFFLOAD_CHECKSUM_F)
 392                         ol_flags |= nix_rx_olflags_get(lookup_mem, w1);
 393         }
 394
 395         if (flag & NIX_RX_OFFLOAD_VLAN_STRIP_F) {
 396                 if (rx->vtag0_gone) {
 397                         ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
 398                         mbuf->vlan_tci = rx->vtag0_tci;
 399                 }
 400                 if (rx->vtag1_gone) {
 401                         ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED;
 402                         mbuf->vlan_tci_outer = rx->vtag1_tci;
 403                 }
 404         }
 405
 406         if (flag & NIX_RX_OFFLOAD_MARK_UPDATE_F)
 407                 ol_flags = nix_update_match_id(rx->match_id, ol_flags, mbuf);
 408
 409         mbuf->ol_flags = ol_flags;
 410         mbuf->pkt_len = len;
 411         mbuf->data_len = len;
 412         *(uint64_t *)(&mbuf->rearm_data) = val;
 413
 414         if (flag & NIX_RX_MULTI_SEG_F)
 415                 /*
 416                  * For multi segment packets, mbuf length correction according
 417                  * to Rx timestamp length will be handled later during
 418                  * timestamp data process.
 419                  * Hence, flag argument is not required.
 420                  */
 421                 nix_cqe_xtract_mseg(rx, mbuf, val, 0);
 422         else
 423                 mbuf->next = NULL;
 424 }
 425
 426 static inline uint16_t
 427 nix_rx_nb_pkts(struct cn10k_eth_rxq *rxq, const uint64_t wdata,
 428                const uint16_t pkts, const uint32_t qmask)
 429 {
 430         uint32_t available = rxq->available;
 431
 432         /* Update the available count if cached value is not enough */
 433         if (unlikely(available < pkts)) {
 434                 uint64_t reg, head, tail;
 435
 436                 /* Use LDADDA version to avoid reorder */
 437                 reg = roc_atomic64_add_sync(wdata, rxq->cq_status);
 438                 /* CQ_OP_STATUS operation error */
 439                 if (reg & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) ||
 440                     reg & BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR))
 441                         return 0;
 442
 443                 tail = reg & 0xFFFFF;
 444                 head = (reg >> 20) & 0xFFFFF;
 445                 if (tail < head)
 446                         available = tail - head + qmask + 1;
 447                 else
 448                         available = tail - head;
 449
 450                 rxq->available = available;
 451         }
 452
 453         return RTE_MIN(pkts, available);
 454 }
 455
 456 static __rte_always_inline void
 457 cn10k_nix_mbuf_to_tstamp(struct rte_mbuf *mbuf,
 458                         struct cnxk_timesync_info *tstamp,
 459                         const uint8_t ts_enable, uint64_t *tstamp_ptr)
 460 {
 461         if (ts_enable) {
 462                 mbuf->pkt_len -= CNXK_NIX_TIMESYNC_RX_OFFSET;
 463                 mbuf->data_len -= CNXK_NIX_TIMESYNC_RX_OFFSET;
 464
 465                 /* Reading the rx timestamp inserted by CGX, viz at
 466                  * starting of the packet data.
 467                  */
 468                 *tstamp_ptr = ((*tstamp_ptr >> 32) * NSEC_PER_SEC) +
 469                         (*tstamp_ptr & 0xFFFFFFFFUL);
 470                 *cnxk_nix_timestamp_dynfield(mbuf, tstamp) =
 471                         rte_be_to_cpu_64(*tstamp_ptr);
 472                 /* RTE_MBUF_F_RX_IEEE1588_TMST flag needs to be set only in case
 473                  * PTP packets are received.
 474                  */
 475                 if (mbuf->packet_type == RTE_PTYPE_L2_ETHER_TIMESYNC) {
 476                         tstamp->rx_tstamp =
 477                                 *cnxk_nix_timestamp_dynfield(mbuf, tstamp);
 478                         tstamp->rx_ready = 1;
 479                         mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP |
 480                                 RTE_MBUF_F_RX_IEEE1588_TMST |
 481                                 tstamp->rx_tstamp_dynflag;
 482                 }
 483         }
 484 }
 485
 486 static __rte_always_inline uint16_t
 487 cn10k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts,
 488                     const uint16_t flags)
 489 {
 490         struct cn10k_eth_rxq *rxq = rx_queue;
 491         const uint64_t mbuf_init = rxq->mbuf_initializer;
 492         const void *lookup_mem = rxq->lookup_mem;
 493         const uint64_t data_off = rxq->data_off;
 494         const uintptr_t desc = rxq->desc;
 495         const uint64_t wdata = rxq->wdata;
 496         const uint32_t qmask = rxq->qmask;
 497         uint64_t lbase = rxq->lmt_base;
 498         uint16_t packets = 0, nb_pkts;
 499         uint8_t loff = 0, lnum = 0;
 500         uint32_t head = rxq->head;
 501         struct nix_cqe_hdr_s *cq;
 502         struct rte_mbuf *mbuf;
 503         uint64_t aura_handle;
 504         uint64_t sa_base;
 505         uint16_t lmt_id;
 506         uint64_t laddr;
 507
 508         nb_pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask);
 509
 510         if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 511                 aura_handle = rxq->aura_handle;
 512                 sa_base = rxq->sa_base;
 513                 sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1);
 514                 ROC_LMT_BASE_ID_GET(lbase, lmt_id);
 515                 laddr = lbase;
 516                 laddr += 8;
 517         }
 518
 519         while (packets < nb_pkts) {
 520                 /* Prefetch N desc ahead */
 521                 rte_prefetch_non_temporal(
 522                         (void *)(desc + (CQE_SZ((head + 2) & qmask))));
 523                 cq = (struct nix_cqe_hdr_s *)(desc + CQE_SZ(head));
 524
 525                 mbuf = nix_get_mbuf_from_cqe(cq, data_off);
 526
 527                 /* Mark mempool obj as "get" as it is alloc'ed by NIX */
 528                 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
 529
 530                 /* Translate meta to mbuf */
 531                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 532                         const uint64_t cq_w1 = *((const uint64_t *)cq + 1);
 533
 534                         mbuf = nix_sec_meta_to_mbuf_sc(cq_w1, sa_base, laddr,
 535                                                        &loff, mbuf, data_off);
 536                 }
 537
 538                 cn10k_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init,
 539                                       flags);
 540                 cn10k_nix_mbuf_to_tstamp(mbuf, rxq->tstamp,
 541                                         (flags & NIX_RX_OFFLOAD_TSTAMP_F),
 542                                         (uint64_t *)((uint8_t *)mbuf
 543                                                                 + data_off));
 544                 rx_pkts[packets++] = mbuf;
 545                 roc_prefetch_store_keep(mbuf);
 546                 head++;
 547                 head &= qmask;
 548
 549                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 550                         /* Flush when we don't have space for 4 meta */
 551                         if ((15 - loff) < 1) {
 552                                 nix_sec_flush_meta(laddr, lmt_id + lnum, loff,
 553                                                    aura_handle);
 554                                 lnum++;
 555                                 lnum &= BIT_ULL(ROC_LMT_LINES_PER_CORE_LOG2) -
 556                                         1;
 557                                 /* First pointer starts at 8B offset */
 558                                 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8);
 559                                 loff = 0;
 560                         }
 561                 }
 562         }
 563
 564         rxq->head = head;
 565         rxq->available -= nb_pkts;
 566
 567         /* Free all the CQs that we've processed */
 568         plt_write64((wdata | nb_pkts), rxq->cq_door);
 569
 570         /* Free remaining meta buffers if any */
 571         if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff) {
 572                 nix_sec_flush_meta(laddr, lmt_id + lnum, loff, aura_handle);
 573                 plt_io_wmb();
 574         }
 575
 576         return nb_pkts;
 577 }
 578
 579 #if defined(RTE_ARCH_ARM64)
 580
 581 static __rte_always_inline uint64_t
 582 nix_vlan_update(const uint64_t w2, uint64_t ol_flags, uint8x16_t *f)
 583 {
 584         if (w2 & BIT_ULL(21) /* vtag0_gone */) {
 585                 ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
 586                 *f = vsetq_lane_u16((uint16_t)(w2 >> 32), *f, 5);
 587         }
 588
 589         return ol_flags;
 590 }
 591
 592 static __rte_always_inline uint64_t
 593 nix_qinq_update(const uint64_t w2, uint64_t ol_flags, struct rte_mbuf *mbuf)
 594 {
 595         if (w2 & BIT_ULL(23) /* vtag1_gone */) {
 596                 ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED;
 597                 mbuf->vlan_tci_outer = (uint16_t)(w2 >> 48);
 598         }
 599
 600         return ol_flags;
 601 }
 602
 603 static __rte_always_inline uint16_t
 604 cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts,
 605                            const uint16_t flags, void *lookup_mem,
 606                            struct cnxk_timesync_info *tstamp,
 607                            uintptr_t lmt_base)
 608 {
 609         struct cn10k_eth_rxq *rxq = args;
 610         const uint64_t mbuf_initializer = (flags & NIX_RX_VWQE_F) ?
 611                                                         *(uint64_t *)args :
 612                                                         rxq->mbuf_initializer;
 613         const uint64x2_t data_off = flags & NIX_RX_VWQE_F ?
 614                                         vdupq_n_u64(RTE_PKTMBUF_HEADROOM) :
 615                                         vdupq_n_u64(rxq->data_off);
 616         const uint32_t qmask = flags & NIX_RX_VWQE_F ? 0 : rxq->qmask;
 617         const uint64_t wdata = flags & NIX_RX_VWQE_F ? 0 : rxq->wdata;
 618         const uintptr_t desc = flags & NIX_RX_VWQE_F ? 0 : rxq->desc;
 619         uint64x2_t cq0_w8, cq1_w8, cq2_w8, cq3_w8, mbuf01, mbuf23;
 620         uint64_t ol_flags0, ol_flags1, ol_flags2, ol_flags3;
 621         uint64x2_t rearm0 = vdupq_n_u64(mbuf_initializer);
 622         uint64x2_t rearm1 = vdupq_n_u64(mbuf_initializer);
 623         uint64x2_t rearm2 = vdupq_n_u64(mbuf_initializer);
 624         uint64x2_t rearm3 = vdupq_n_u64(mbuf_initializer);
 625         struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3;
 626         uint64_t aura_handle, lbase, laddr;
 627         uint8_t loff = 0, lnum = 0, shft = 0;
 628         uint8x16_t f0, f1, f2, f3;
 629         uint16_t lmt_id, d_off;
 630         uint16_t packets = 0;
 631         uint16_t pkts_left;
 632         uintptr_t sa_base;
 633         uint32_t head;
 634         uintptr_t cq0;
 635
 636         if (!(flags & NIX_RX_VWQE_F)) {
 637                 lookup_mem = rxq->lookup_mem;
 638                 head = rxq->head;
 639
 640                 pkts = nix_rx_nb_pkts(rxq, wdata, pkts, qmask);
 641                 pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1);
 642                 /* Packets has to be floor-aligned to NIX_DESCS_PER_LOOP */
 643                 pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
 644                 if (flags & NIX_RX_OFFLOAD_TSTAMP_F)
 645                         tstamp = rxq->tstamp;
 646         } else {
 647                 RTE_SET_USED(head);
 648         }
 649
 650         if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 651                 if (flags & NIX_RX_VWQE_F) {
 652                         uint16_t port;
 653
 654                         mbuf0 = (struct rte_mbuf *)((uintptr_t)mbufs[0] -
 655                                                     sizeof(struct rte_mbuf));
 656                         /* Pick first mbuf's aura handle assuming all
 657                          * mbufs are from a vec and are from same RQ.
 658                          */
 659                         aura_handle = mbuf0->pool->pool_id;
 660                         /* Calculate offset from mbuf to actual data area */
 661                         d_off = ((uintptr_t)mbuf0->buf_addr - (uintptr_t)mbuf0);
 662                         d_off += (mbuf_initializer & 0xFFFF);
 663
 664                         /* Get SA Base from lookup tbl using port_id */
 665                         port = mbuf_initializer >> 48;
 666                         sa_base = cnxk_nix_sa_base_get(port, lookup_mem);
 667
 668                         lbase = lmt_base;
 669                 } else {
 670                         aura_handle = rxq->aura_handle;
 671                         d_off = rxq->data_off;
 672                         sa_base = rxq->sa_base;
 673                         lbase = rxq->lmt_base;
 674                 }
 675                 sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1);
 676                 ROC_LMT_BASE_ID_GET(lbase, lmt_id);
 677                 lnum = 0;
 678                 laddr = lbase;
 679                 laddr += 8;
 680         }
 681
 682         while (packets < pkts) {
 683                 if (!(flags & NIX_RX_VWQE_F)) {
 684                         /* Exit loop if head is about to wrap and become
 685                          * unaligned.
 686                          */
 687                         if (((head + NIX_DESCS_PER_LOOP - 1) & qmask) <
 688                             NIX_DESCS_PER_LOOP) {
 689                                 pkts_left += (pkts - packets);
 690                                 break;
 691                         }
 692
 693                         cq0 = desc + CQE_SZ(head);
 694                 } else {
 695                         cq0 = (uintptr_t)&mbufs[packets];
 696                 }
 697
 698                 if (flags & NIX_RX_VWQE_F) {
 699                         if (pkts - packets > 4) {
 700                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
 701                                         4, 0, flags));
 702                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
 703                                         5, 0, flags));
 704                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
 705                                         6, 0, flags));
 706                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0,
 707                                         7, 0, flags));
 708
 709                                 if (likely(pkts - packets > 8)) {
 710                                         rte_prefetch1(CQE_PTR_OFF(cq0,
 711                                                 8, 0, flags));
 712                                         rte_prefetch1(CQE_PTR_OFF(cq0,
 713                                                 9, 0, flags));
 714                                         rte_prefetch1(CQE_PTR_OFF(cq0,
 715                                                 10, 0, flags));
 716                                         rte_prefetch1(CQE_PTR_OFF(cq0,
 717                                                 11, 0, flags));
 718                                         if (pkts - packets > 12) {
 719                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
 720                                                         12, 0, flags));
 721                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
 722                                                         13, 0, flags));
 723                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
 724                                                         14, 0, flags));
 725                                                 rte_prefetch1(CQE_PTR_OFF(cq0,
 726                                                         15, 0, flags));
 727                                         }
 728                                 }
 729
 730                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
 731                                         4, RTE_PKTMBUF_HEADROOM, flags));
 732                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
 733                                         5, RTE_PKTMBUF_HEADROOM, flags));
 734                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
 735                                         6, RTE_PKTMBUF_HEADROOM, flags));
 736                                 rte_prefetch0(CQE_PTR_DIFF(cq0,
 737                                         7, RTE_PKTMBUF_HEADROOM, flags));
 738
 739                                 if (likely(pkts - packets > 8)) {
 740                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
 741                                                 8, RTE_PKTMBUF_HEADROOM, flags));
 742                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
 743                                                 9, RTE_PKTMBUF_HEADROOM, flags));
 744                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
 745                                                 10, RTE_PKTMBUF_HEADROOM, flags));
 746                                         rte_prefetch0(CQE_PTR_DIFF(cq0,
 747                                                 11, RTE_PKTMBUF_HEADROOM, flags));
 748                                 }
 749                         }
 750                 } else {
 751                         if (pkts - packets > 4) {
 752                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 4, 64, flags));
 753                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 5, 64, flags));
 754                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 6, 64, flags));
 755                                 rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 7, 64, flags));
 756                         }
 757                 }
 758
 759                 if (!(flags & NIX_RX_VWQE_F)) {
 760                         /* Get NIX_RX_SG_S for size and buffer pointer */
 761                         cq0_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 0, 64, flags));
 762                         cq1_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 1, 64, flags));
 763                         cq2_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 2, 64, flags));
 764                         cq3_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 3, 64, flags));
 765
 766                         /* Extract mbuf from NIX_RX_SG_S */
 767                         mbuf01 = vzip2q_u64(cq0_w8, cq1_w8);
 768                         mbuf23 = vzip2q_u64(cq2_w8, cq3_w8);
 769                         mbuf01 = vqsubq_u64(mbuf01, data_off);
 770                         mbuf23 = vqsubq_u64(mbuf23, data_off);
 771                 } else {
 772                         mbuf01 =
 773                                 vsubq_u64(vld1q_u64((uint64_t *)cq0), data_off);
 774                         mbuf23 = vsubq_u64(vld1q_u64((uint64_t *)(cq0 + 16)),
 775                                            data_off);
 776                 }
 777
 778                 /* Move mbufs to scalar registers for future use */
 779                 mbuf0 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 0);
 780                 mbuf1 = (struct rte_mbuf *)vgetq_lane_u64(mbuf01, 1);
 781                 mbuf2 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 0);
 782                 mbuf3 = (struct rte_mbuf *)vgetq_lane_u64(mbuf23, 1);
 783
 784                 if (!(flags & NIX_RX_VWQE_F)) {
 785                         /* Mask to get packet len from NIX_RX_SG_S */
 786                         const uint8x16_t shuf_msk = {
 787                                 0xFF, 0xFF, /* pkt_type set as unknown */
 788                                 0xFF, 0xFF, /* pkt_type set as unknown */
 789                                 0,    1,    /* octet 1~0, low 16 bits pkt_len */
 790                                 0xFF, 0xFF, /* skip high 16it pkt_len, zero out */
 791                                 0,    1,    /* octet 1~0, 16 bits data_len */
 792                                 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
 793
 794                         /* Form the rx_descriptor_fields1 with pkt_len and data_len */
 795                         f0 = vqtbl1q_u8(cq0_w8, shuf_msk);
 796                         f1 = vqtbl1q_u8(cq1_w8, shuf_msk);
 797                         f2 = vqtbl1q_u8(cq2_w8, shuf_msk);
 798                         f3 = vqtbl1q_u8(cq3_w8, shuf_msk);
 799                 }
 800                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 801                         /* Prefetch probable CPT parse header area */
 802                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf0, d_off));
 803                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf1, d_off));
 804                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf2, d_off));
 805                         rte_prefetch_non_temporal(RTE_PTR_ADD(mbuf3, d_off));
 806                 }
 807
 808                 /* Load CQE word0 and word 1 */
 809                 const uint64_t cq0_w0 = *CQE_PTR_OFF(cq0, 0, 0, flags);
 810                 const uint64_t cq0_w1 = *CQE_PTR_OFF(cq0, 0, 8, flags);
 811                 const uint64_t cq0_w2 = *CQE_PTR_OFF(cq0, 0, 16, flags);
 812                 const uint64_t cq1_w0 = *CQE_PTR_OFF(cq0, 1, 0, flags);
 813                 const uint64_t cq1_w1 = *CQE_PTR_OFF(cq0, 1, 8, flags);
 814                 const uint64_t cq1_w2 = *CQE_PTR_OFF(cq0, 1, 16, flags);
 815                 const uint64_t cq2_w0 = *CQE_PTR_OFF(cq0, 2, 0, flags);
 816                 const uint64_t cq2_w1 = *CQE_PTR_OFF(cq0, 2, 8, flags);
 817                 const uint64_t cq2_w2 = *CQE_PTR_OFF(cq0, 2, 16, flags);
 818                 const uint64_t cq3_w0 = *CQE_PTR_OFF(cq0, 3, 0, flags);
 819                 const uint64_t cq3_w1 = *CQE_PTR_OFF(cq0, 3, 8, flags);
 820                 const uint64_t cq3_w2 = *CQE_PTR_OFF(cq0, 3, 16, flags);
 821
 822                 if (flags & NIX_RX_VWQE_F) {
 823                         uint16_t psize0, psize1, psize2, psize3;
 824
 825                         psize0 = (cq0_w2 & 0xFFFF) + 1;
 826                         psize1 = (cq1_w2 & 0xFFFF) + 1;
 827                         psize2 = (cq2_w2 & 0xFFFF) + 1;
 828                         psize3 = (cq3_w2 & 0xFFFF) + 1;
 829
 830                         f0 = vdupq_n_u64(0);
 831                         f1 = vdupq_n_u64(0);
 832                         f2 = vdupq_n_u64(0);
 833                         f3 = vdupq_n_u64(0);
 834
 835                         f0 = vsetq_lane_u16(psize0, f0, 2);
 836                         f0 = vsetq_lane_u16(psize0, f0, 4);
 837
 838                         f1 = vsetq_lane_u16(psize1, f1, 2);
 839                         f1 = vsetq_lane_u16(psize1, f1, 4);
 840
 841                         f2 = vsetq_lane_u16(psize2, f2, 2);
 842                         f2 = vsetq_lane_u16(psize2, f2, 4);
 843
 844                         f3 = vsetq_lane_u16(psize3, f3, 2);
 845                         f3 = vsetq_lane_u16(psize3, f3, 4);
 846                 }
 847
 848                 if (flags & NIX_RX_OFFLOAD_RSS_F) {
 849                         /* Fill rss in the rx_descriptor_fields1 */
 850                         f0 = vsetq_lane_u32(cq0_w0, f0, 3);
 851                         f1 = vsetq_lane_u32(cq1_w0, f1, 3);
 852                         f2 = vsetq_lane_u32(cq2_w0, f2, 3);
 853                         f3 = vsetq_lane_u32(cq3_w0, f3, 3);
 854                         ol_flags0 = RTE_MBUF_F_RX_RSS_HASH;
 855                         ol_flags1 = RTE_MBUF_F_RX_RSS_HASH;
 856                         ol_flags2 = RTE_MBUF_F_RX_RSS_HASH;
 857                         ol_flags3 = RTE_MBUF_F_RX_RSS_HASH;
 858                 } else {
 859                         ol_flags0 = 0;
 860                         ol_flags1 = 0;
 861                         ol_flags2 = 0;
 862                         ol_flags3 = 0;
 863                 }
 864
 865                 if (flags & NIX_RX_OFFLOAD_PTYPE_F) {
 866                         /* Fill packet_type in the rx_descriptor_fields1 */
 867                         f0 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq0_w1),
 868                                             f0, 0);
 869                         f1 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq1_w1),
 870                                             f1, 0);
 871                         f2 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq2_w1),
 872                                             f2, 0);
 873                         f3 = vsetq_lane_u32(nix_ptype_get(lookup_mem, cq3_w1),
 874                                             f3, 0);
 875                 }
 876
 877                 if (flags & NIX_RX_OFFLOAD_CHECKSUM_F) {
 878                         ol_flags0 |= nix_rx_olflags_get(lookup_mem, cq0_w1);
 879                         ol_flags1 |= nix_rx_olflags_get(lookup_mem, cq1_w1);
 880                         ol_flags2 |= nix_rx_olflags_get(lookup_mem, cq2_w1);
 881                         ol_flags3 |= nix_rx_olflags_get(lookup_mem, cq3_w1);
 882                 }
 883
 884                 /* Mark mempool obj as "get" as it is alloc'ed by NIX */
 885                 RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1);
 886                 RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1);
 887                 RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1);
 888                 RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1);
 889
 890                 /* Translate meta to mbuf */
 891                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
 892                         /* Checksum ol_flags will be cleared if mbuf is meta */
 893                         mbuf0 = nix_sec_meta_to_mbuf(cq0_w1, sa_base, laddr,
 894                                                      &loff, mbuf0, d_off, &f0,
 895                                                      &ol_flags0);
 896                         mbuf01 = vsetq_lane_u64((uint64_t)mbuf0, mbuf01, 0);
 897
 898                         mbuf1 = nix_sec_meta_to_mbuf(cq1_w1, sa_base, laddr,
 899                                                      &loff, mbuf1, d_off, &f1,
 900                                                      &ol_flags1);
 901                         mbuf01 = vsetq_lane_u64((uint64_t)mbuf1, mbuf01, 1);
 902
 903                         mbuf2 = nix_sec_meta_to_mbuf(cq2_w1, sa_base, laddr,
 904                                                      &loff, mbuf2, d_off, &f2,
 905                                                      &ol_flags2);
 906                         mbuf23 = vsetq_lane_u64((uint64_t)mbuf2, mbuf23, 0);
 907
 908                         mbuf3 = nix_sec_meta_to_mbuf(cq3_w1, sa_base, laddr,
 909                                                      &loff, mbuf3, d_off, &f3,
 910                                                      &ol_flags3);
 911                         mbuf23 = vsetq_lane_u64((uint64_t)mbuf3, mbuf23, 1);
 912                 }
 913
 914                 if (flags & NIX_RX_OFFLOAD_VLAN_STRIP_F) {
 915
 916                         ol_flags0 = nix_vlan_update(cq0_w2, ol_flags0, &f0);
 917                         ol_flags1 = nix_vlan_update(cq1_w2, ol_flags1, &f1);
 918                         ol_flags2 = nix_vlan_update(cq2_w2, ol_flags2, &f2);
 919                         ol_flags3 = nix_vlan_update(cq3_w2, ol_flags3, &f3);
 920
 921                         ol_flags0 = nix_qinq_update(cq0_w2, ol_flags0, mbuf0);
 922                         ol_flags1 = nix_qinq_update(cq1_w2, ol_flags1, mbuf1);
 923                         ol_flags2 = nix_qinq_update(cq2_w2, ol_flags2, mbuf2);
 924                         ol_flags3 = nix_qinq_update(cq3_w2, ol_flags3, mbuf3);
 925                 }
 926
 927                 if (flags & NIX_RX_OFFLOAD_MARK_UPDATE_F) {
 928                         ol_flags0 = nix_update_match_id(
 929                                 *(uint16_t *)CQE_PTR_OFF(cq0, 0, 38, flags),
 930                                 ol_flags0, mbuf0);
 931                         ol_flags1 = nix_update_match_id(
 932                                 *(uint16_t *)CQE_PTR_OFF(cq0, 1, 38, flags),
 933                                 ol_flags1, mbuf1);
 934                         ol_flags2 = nix_update_match_id(
 935                                 *(uint16_t *)CQE_PTR_OFF(cq0, 2, 38, flags),
 936                                 ol_flags2, mbuf2);
 937                         ol_flags3 = nix_update_match_id(
 938                                 *(uint16_t *)CQE_PTR_OFF(cq0, 3, 38, flags),
 939                                 ol_flags3, mbuf3);
 940                 }
 941
 942                 if (flags & NIX_RX_OFFLOAD_TSTAMP_F) {
 943                         const uint16x8_t len_off = {
 944                                 0,                           /* ptype   0:15 */
 945                                 0,                           /* ptype  16:32 */
 946                                 CNXK_NIX_TIMESYNC_RX_OFFSET, /* pktlen  0:15*/
 947                                 0,                           /* pktlen 16:32 */
 948                                 CNXK_NIX_TIMESYNC_RX_OFFSET, /* datalen 0:15 */
 949                                 0,
 950                                 0,
 951                                 0};
 952                         const uint32x4_t ptype = {RTE_PTYPE_L2_ETHER_TIMESYNC,
 953                                                   RTE_PTYPE_L2_ETHER_TIMESYNC,
 954                                                   RTE_PTYPE_L2_ETHER_TIMESYNC,
 955                                                   RTE_PTYPE_L2_ETHER_TIMESYNC};
 956                         const uint64_t ts_olf = RTE_MBUF_F_RX_IEEE1588_PTP |
 957                                                 RTE_MBUF_F_RX_IEEE1588_TMST |
 958                                                 tstamp->rx_tstamp_dynflag;
 959                         const uint32x4_t and_mask = {0x1, 0x2, 0x4, 0x8};
 960                         uint64x2_t ts01, ts23, mask;
 961                         uint64_t ts[4];
 962                         uint8_t res;
 963
 964                         /* Subtract timesync length from total pkt length. */
 965                         f0 = vsubq_u16(f0, len_off);
 966                         f1 = vsubq_u16(f1, len_off);
 967                         f2 = vsubq_u16(f2, len_off);
 968                         f3 = vsubq_u16(f3, len_off);
 969
 970                         /* Get the address of actual timestamp. */
 971                         ts01 = vaddq_u64(mbuf01, data_off);
 972                         ts23 = vaddq_u64(mbuf23, data_off);
 973                         /* Load timestamp from address. */
 974                         ts01 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts01,
 975                                                                           0),
 976                                               ts01, 0);
 977                         ts01 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts01,
 978                                                                           1),
 979                                               ts01, 1);
 980                         ts23 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts23,
 981                                                                           0),
 982                                               ts23, 0);
 983                         ts23 = vsetq_lane_u64(*(uint64_t *)vgetq_lane_u64(ts23,
 984                                                                           1),
 985                                               ts23, 1);
 986                         /* Convert from be to cpu byteorder. */
 987                         ts01 = vrev64q_u8(ts01);
 988                         ts23 = vrev64q_u8(ts23);
 989                         /* Store timestamp into scalar for later use. */
 990                         ts[0] = vgetq_lane_u64(ts01, 0);
 991                         ts[1] = vgetq_lane_u64(ts01, 1);
 992                         ts[2] = vgetq_lane_u64(ts23, 0);
 993                         ts[3] = vgetq_lane_u64(ts23, 1);
 994
 995                         /* Store timestamp into dynfield. */
 996                         *cnxk_nix_timestamp_dynfield(mbuf0, tstamp) = ts[0];
 997                         *cnxk_nix_timestamp_dynfield(mbuf1, tstamp) = ts[1];
 998                         *cnxk_nix_timestamp_dynfield(mbuf2, tstamp) = ts[2];
 999                         *cnxk_nix_timestamp_dynfield(mbuf3, tstamp) = ts[3];
1000
1001                         /* Generate ptype mask to filter L2 ether timesync */
1002                         mask = vdupq_n_u32(vgetq_lane_u32(f0, 0));
1003                         mask = vsetq_lane_u32(vgetq_lane_u32(f1, 0), mask, 1);
1004                         mask = vsetq_lane_u32(vgetq_lane_u32(f2, 0), mask, 2);
1005                         mask = vsetq_lane_u32(vgetq_lane_u32(f3, 0), mask, 3);
1006
1007                         /* Match against L2 ether timesync. */
1008                         mask = vceqq_u32(mask, ptype);
1009                         /* Convert from vector from scalar mask */
1010                         res = vaddvq_u32(vandq_u32(mask, and_mask));
1011                         res &= 0xF;
1012
1013                         if (res) {
1014                                 /* Fill in the ol_flags for any packets that
1015                                  * matched.
1016                                  */
1017                                 ol_flags0 |= ((res & 0x1) ? ts_olf : 0);
1018                                 ol_flags1 |= ((res & 0x2) ? ts_olf : 0);
1019                                 ol_flags2 |= ((res & 0x4) ? ts_olf : 0);
1020                                 ol_flags3 |= ((res & 0x8) ? ts_olf : 0);
1021
1022                                 /* Update Rxq timestamp with the latest
1023                                  * timestamp.
1024                                  */
1025                                 tstamp->rx_ready = 1;
1026                                 tstamp->rx_tstamp = ts[31 - __builtin_clz(res)];
1027                         }
1028                 }
1029
1030                 /* Form rearm_data with ol_flags */
1031                 rearm0 = vsetq_lane_u64(ol_flags0, rearm0, 1);
1032                 rearm1 = vsetq_lane_u64(ol_flags1, rearm1, 1);
1033                 rearm2 = vsetq_lane_u64(ol_flags2, rearm2, 1);
1034                 rearm3 = vsetq_lane_u64(ol_flags3, rearm3, 1);
1035
1036                 /* Update rx_descriptor_fields1 */
1037                 vst1q_u64((uint64_t *)mbuf0->rx_descriptor_fields1, f0);
1038                 vst1q_u64((uint64_t *)mbuf1->rx_descriptor_fields1, f1);
1039                 vst1q_u64((uint64_t *)mbuf2->rx_descriptor_fields1, f2);
1040                 vst1q_u64((uint64_t *)mbuf3->rx_descriptor_fields1, f3);
1041
1042                 /* Update rearm_data */
1043                 vst1q_u64((uint64_t *)mbuf0->rearm_data, rearm0);
1044                 vst1q_u64((uint64_t *)mbuf1->rearm_data, rearm1);
1045                 vst1q_u64((uint64_t *)mbuf2->rearm_data, rearm2);
1046                 vst1q_u64((uint64_t *)mbuf3->rearm_data, rearm3);
1047
1048                 /* Store the mbufs to rx_pkts */
1049                 vst1q_u64((uint64_t *)&mbufs[packets], mbuf01);
1050                 vst1q_u64((uint64_t *)&mbufs[packets + 2], mbuf23);
1051
1052                 if (flags & NIX_RX_MULTI_SEG_F) {
1053                         /* Multi segment is enable build mseg list for
1054                          * individual mbufs in scalar mode.
1055                          */
1056                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1057                                             (CQE_PTR_OFF(cq0, 0, 8, flags)),
1058                                             mbuf0, mbuf_initializer, flags);
1059                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1060                                             (CQE_PTR_OFF(cq0, 1, 8, flags)),
1061                                             mbuf1, mbuf_initializer, flags);
1062                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1063                                             (CQE_PTR_OFF(cq0, 2, 8, flags)),
1064                                             mbuf2, mbuf_initializer, flags);
1065                         nix_cqe_xtract_mseg((union nix_rx_parse_u *)
1066                                             (CQE_PTR_OFF(cq0, 3, 8, flags)),
1067                                             mbuf3, mbuf_initializer, flags);
1068                 }
1069
1070                 /* Mark mempool obj as "get" as it is alloc'ed by NIX */
1071                 RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1);
1072                 RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1);
1073                 RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1);
1074                 RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1);
1075
1076                 nix_mbuf_validate_next(mbuf0);
1077                 nix_mbuf_validate_next(mbuf1);
1078                 nix_mbuf_validate_next(mbuf2);
1079                 nix_mbuf_validate_next(mbuf3);
1080
1081                 packets += NIX_DESCS_PER_LOOP;
1082
1083                 if (!(flags & NIX_RX_VWQE_F)) {
1084                         /* Advance head pointer and packets */
1085                         head += NIX_DESCS_PER_LOOP;
1086                         head &= qmask;
1087                 }
1088
1089                 if (flags & NIX_RX_OFFLOAD_SECURITY_F) {
1090                         /* Check if lmtline border is crossed and adjust lnum */
1091                         if (loff > 15) {
1092                                 /* Update aura handle */
1093                                 *(uint64_t *)(laddr - 8) =
1094                                         (((uint64_t)(15 & 0x1) << 32) |
1095                                     roc_npa_aura_handle_to_aura(aura_handle));
1096                                 loff = loff - 15;
1097                                 shft += 3;
1098
1099                                 lnum++;
1100                                 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8);
1101                                 /* Pick the pointer from 16th index and put it
1102                                  * at end of this new line.
1103                                  */
1104                                 *(uint64_t *)(laddr + (loff << 3) - 8) =
1105                                         *(uint64_t *)(laddr - 8);
1106                         }
1107
1108                         /* Flush it when we are in 16th line and might
1109                          * overflow it
1110                          */
1111                         if (lnum >= 15 && loff >= 12) {
1112                                 /* 16 LMT Line size m1 */
1113                                 uint64_t data = BIT_ULL(48) - 1;
1114
1115                                 /* Update aura handle */
1116                                 *(uint64_t *)(laddr - 8) =
1117                                         (((uint64_t)(loff & 0x1) << 32) |
1118                                     roc_npa_aura_handle_to_aura(aura_handle));
1119
1120                                 data = (data & ~(0x7UL << shft)) |
1121                                        (((uint64_t)loff >> 1) << shft);
1122
1123                                 /* Send up to 16 lmt lines of pointers */
1124                                 nix_sec_flush_meta_burst(lmt_id, data, lnum + 1,
1125                                                          aura_handle);
1126                                 rte_io_wmb();
1127                                 lnum = 0;
1128                                 loff = 0;
1129                                 shft = 0;
1130                                 /* First pointer starts at 8B offset */
1131                                 laddr = (uintptr_t)LMT_OFF(lbase, lnum, 8);
1132                         }
1133                 }
1134         }
1135
1136         if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff) {
1137                 /* 16 LMT Line size m1 */
1138                 uint64_t data = BIT_ULL(48) - 1;
1139
1140                 /* Update aura handle */
1141                 *(uint64_t *)(laddr - 8) =
1142                         (((uint64_t)(loff & 0x1) << 32) |
1143                          roc_npa_aura_handle_to_aura(aura_handle));
1144
1145                 data = (data & ~(0x7UL << shft)) |
1146                        (((uint64_t)loff >> 1) << shft);
1147
1148                 /* Send up to 16 lmt lines of pointers */
1149                 nix_sec_flush_meta_burst(lmt_id, data, lnum + 1, aura_handle);
1150                 if (flags & NIX_RX_VWQE_F)
1151                         plt_io_wmb();
1152         }
1153
1154         if (flags & NIX_RX_VWQE_F)
1155                 return packets;
1156
1157         rxq->head = head;
1158         rxq->available -= packets;
1159
1160         rte_io_wmb();
1161         /* Free all the CQs that we've processed */
1162         plt_write64((rxq->wdata | packets), rxq->cq_door);
1163
1164         if (unlikely(pkts_left))
1165                 packets += cn10k_nix_recv_pkts(args, &mbufs[packets], pkts_left,
1166                                                flags);
1167
1168         return packets;
1169 }
1170
1171 #else
1172
1173 static inline uint16_t
1174 cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts,
1175                            const uint16_t flags, void *lookup_mem,
1176                            struct cnxk_timesync_info *tstamp,
1177                            uintptr_t lmt_base)
1178 {
1179         RTE_SET_USED(args);
1180         RTE_SET_USED(mbufs);
1181         RTE_SET_USED(pkts);
1182         RTE_SET_USED(flags);
1183         RTE_SET_USED(lookup_mem);
1184         RTE_SET_USED(tstamp);
1185         RTE_SET_USED(lmt_base);
1186
1187         return 0;
1188 }
1189
1190 #endif
1191
1192
1193 #define RSS_F     NIX_RX_OFFLOAD_RSS_F
1194 #define PTYPE_F   NIX_RX_OFFLOAD_PTYPE_F
1195 #define CKSUM_F   NIX_RX_OFFLOAD_CHECKSUM_F
1196 #define MARK_F    NIX_RX_OFFLOAD_MARK_UPDATE_F
1197 #define TS_F      NIX_RX_OFFLOAD_TSTAMP_F
1198 #define RX_VLAN_F NIX_RX_OFFLOAD_VLAN_STRIP_F
1199 #define R_SEC_F   NIX_RX_OFFLOAD_SECURITY_F
1200
1201 /* [R_SEC_F] [RX_VLAN_F] [TS] [MARK] [CKSUM] [PTYPE] [RSS] */
1202 #define NIX_RX_FASTPATH_MODES_0_15                                             \
1203         R(no_offload, NIX_RX_OFFLOAD_NONE)                                     \
1204         R(rss, RSS_F)                                                          \
1205         R(ptype, PTYPE_F)                                                      \
1206         R(ptype_rss, PTYPE_F | RSS_F)                                          \
1207         R(cksum, CKSUM_F)                                                      \
1208         R(cksum_rss, CKSUM_F | RSS_F)                                          \
1209         R(cksum_ptype, CKSUM_F | PTYPE_F)                                      \
1210         R(cksum_ptype_rss, CKSUM_F | PTYPE_F | RSS_F)                          \
1211         R(mark, MARK_F)                                                        \
1212         R(mark_rss, MARK_F | RSS_F)                                            \
1213         R(mark_ptype, MARK_F | PTYPE_F)                                        \
1214         R(mark_ptype_rss, MARK_F | PTYPE_F | RSS_F)                            \
1215         R(mark_cksum, MARK_F | CKSUM_F)                                        \
1216         R(mark_cksum_rss, MARK_F | CKSUM_F | RSS_F)                            \
1217         R(mark_cksum_ptype, MARK_F | CKSUM_F | PTYPE_F)                        \
1218         R(mark_cksum_ptype_rss, MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1219
1220 #define NIX_RX_FASTPATH_MODES_16_31                                            \
1221         R(ts, TS_F)                                                            \
1222         R(ts_rss, TS_F | RSS_F)                                                \
1223         R(ts_ptype, TS_F | PTYPE_F)                                            \
1224         R(ts_ptype_rss, TS_F | PTYPE_F | RSS_F)                                \
1225         R(ts_cksum, TS_F | CKSUM_F)                                            \
1226         R(ts_cksum_rss, TS_F | CKSUM_F | RSS_F)                                \
1227         R(ts_cksum_ptype, TS_F | CKSUM_F | PTYPE_F)                            \
1228         R(ts_cksum_ptype_rss, TS_F | CKSUM_F | PTYPE_F | RSS_F)                \
1229         R(ts_mark, TS_F | MARK_F)                                              \
1230         R(ts_mark_rss, TS_F | MARK_F | RSS_F)                                  \
1231         R(ts_mark_ptype, TS_F | MARK_F | PTYPE_F)                              \
1232         R(ts_mark_ptype_rss, TS_F | MARK_F | PTYPE_F | RSS_F)                  \
1233         R(ts_mark_cksum, TS_F | MARK_F | CKSUM_F)                              \
1234         R(ts_mark_cksum_rss, TS_F | MARK_F | CKSUM_F | RSS_F)                  \
1235         R(ts_mark_cksum_ptype, TS_F | MARK_F | CKSUM_F | PTYPE_F)              \
1236         R(ts_mark_cksum_ptype_rss, TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1237
1238 #define NIX_RX_FASTPATH_MODES_32_47                                            \
1239         R(vlan, RX_VLAN_F)                                                     \
1240         R(vlan_rss, RX_VLAN_F | RSS_F)                                         \
1241         R(vlan_ptype, RX_VLAN_F | PTYPE_F)                                     \
1242         R(vlan_ptype_rss, RX_VLAN_F | PTYPE_F | RSS_F)                         \
1243         R(vlan_cksum, RX_VLAN_F | CKSUM_F)                                     \
1244         R(vlan_cksum_rss, RX_VLAN_F | CKSUM_F | RSS_F)                         \
1245         R(vlan_cksum_ptype, RX_VLAN_F | CKSUM_F | PTYPE_F)                     \
1246         R(vlan_cksum_ptype_rss, RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)         \
1247         R(vlan_mark, RX_VLAN_F | MARK_F)                                       \
1248         R(vlan_mark_rss, RX_VLAN_F | MARK_F | RSS_F)                           \
1249         R(vlan_mark_ptype, RX_VLAN_F | MARK_F | PTYPE_F)                       \
1250         R(vlan_mark_ptype_rss, RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)           \
1251         R(vlan_mark_cksum, RX_VLAN_F | MARK_F | CKSUM_F)                       \
1252         R(vlan_mark_cksum_rss, RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)           \
1253         R(vlan_mark_cksum_ptype, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)       \
1254         R(vlan_mark_cksum_ptype_rss,                                           \
1255           RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1256
1257 #define NIX_RX_FASTPATH_MODES_48_63                                            \
1258         R(vlan_ts, RX_VLAN_F | TS_F)                                           \
1259         R(vlan_ts_rss, RX_VLAN_F | TS_F | RSS_F)                               \
1260         R(vlan_ts_ptype, RX_VLAN_F | TS_F | PTYPE_F)                           \
1261         R(vlan_ts_ptype_rss, RX_VLAN_F | TS_F | PTYPE_F | RSS_F)               \
1262         R(vlan_ts_cksum, RX_VLAN_F | TS_F | CKSUM_F)                           \
1263         R(vlan_ts_cksum_rss, RX_VLAN_F | TS_F | CKSUM_F | RSS_F)               \
1264         R(vlan_ts_cksum_ptype, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)           \
1265         R(vlan_ts_cksum_ptype_rss,                                             \
1266           RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)                        \
1267         R(vlan_ts_mark, RX_VLAN_F | TS_F | MARK_F)                             \
1268         R(vlan_ts_mark_rss, RX_VLAN_F | TS_F | MARK_F | RSS_F)                 \
1269         R(vlan_ts_mark_ptype, RX_VLAN_F | TS_F | MARK_F | PTYPE_F)             \
1270         R(vlan_ts_mark_ptype_rss, RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
1271         R(vlan_ts_mark_cksum, RX_VLAN_F | TS_F | MARK_F | CKSUM_F)             \
1272         R(vlan_ts_mark_cksum_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
1273         R(vlan_ts_mark_cksum_ptype,                                            \
1274           RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                       \
1275         R(vlan_ts_mark_cksum_ptype_rss,                                        \
1276           RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1277
1278 #define NIX_RX_FASTPATH_MODES_64_79                                            \
1279         R(sec, R_SEC_F)                                                        \
1280         R(sec_rss, R_SEC_F | RSS_F)                                            \
1281         R(sec_ptype, R_SEC_F | PTYPE_F)                                        \
1282         R(sec_ptype_rss, R_SEC_F | PTYPE_F | RSS_F)                            \
1283         R(sec_cksum, R_SEC_F | CKSUM_F)                                        \
1284         R(sec_cksum_rss, R_SEC_F | CKSUM_F | RSS_F)                            \
1285         R(sec_cksum_ptype, R_SEC_F | CKSUM_F | PTYPE_F)                        \
1286         R(sec_cksum_ptype_rss, R_SEC_F | CKSUM_F | PTYPE_F | RSS_F)            \
1287         R(sec_mark, R_SEC_F | MARK_F)                                          \
1288         R(sec_mark_rss, R_SEC_F | MARK_F | RSS_F)                              \
1289         R(sec_mark_ptype, R_SEC_F | MARK_F | PTYPE_F)                          \
1290         R(sec_mark_ptype_rss, R_SEC_F | MARK_F | PTYPE_F | RSS_F)              \
1291         R(sec_mark_cksum, R_SEC_F | MARK_F | CKSUM_F)                          \
1292         R(sec_mark_cksum_rss, R_SEC_F | MARK_F | CKSUM_F | RSS_F)              \
1293         R(sec_mark_cksum_ptype, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F)          \
1294         R(sec_mark_cksum_ptype_rss,                                            \
1295           R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1296
1297 #define NIX_RX_FASTPATH_MODES_80_95                                            \
1298         R(sec_ts, R_SEC_F | TS_F)                                              \
1299         R(sec_ts_rss, R_SEC_F | TS_F | RSS_F)                                  \
1300         R(sec_ts_ptype, R_SEC_F | TS_F | PTYPE_F)                              \
1301         R(sec_ts_ptype_rss, R_SEC_F | TS_F | PTYPE_F | RSS_F)                  \
1302         R(sec_ts_cksum, R_SEC_F | TS_F | CKSUM_F)                              \
1303         R(sec_ts_cksum_rss, R_SEC_F | TS_F | CKSUM_F | RSS_F)                  \
1304         R(sec_ts_cksum_ptype, R_SEC_F | TS_F | CKSUM_F | PTYPE_F)              \
1305         R(sec_ts_cksum_ptype_rss, R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)  \
1306         R(sec_ts_mark, R_SEC_F | TS_F | MARK_F)                                \
1307         R(sec_ts_mark_rss, R_SEC_F | TS_F | MARK_F | RSS_F)                    \
1308         R(sec_ts_mark_ptype, R_SEC_F | TS_F | MARK_F | PTYPE_F)                \
1309         R(sec_ts_mark_ptype_rss, R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F)    \
1310         R(sec_ts_mark_cksum, R_SEC_F | TS_F | MARK_F | CKSUM_F)                \
1311         R(sec_ts_mark_cksum_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F)    \
1312         R(sec_ts_mark_cksum_ptype,                                             \
1313           R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                         \
1314         R(sec_ts_mark_cksum_ptype_rss,                                         \
1315           R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1316
1317 #define NIX_RX_FASTPATH_MODES_96_111                                           \
1318         R(sec_vlan, R_SEC_F | RX_VLAN_F)                                       \
1319         R(sec_vlan_rss, R_SEC_F | RX_VLAN_F | RSS_F)                           \
1320         R(sec_vlan_ptype, R_SEC_F | RX_VLAN_F | PTYPE_F)                       \
1321         R(sec_vlan_ptype_rss, R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F)           \
1322         R(sec_vlan_cksum, R_SEC_F | RX_VLAN_F | CKSUM_F)                       \
1323         R(sec_vlan_cksum_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F)           \
1324         R(sec_vlan_cksum_ptype, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F)       \
1325         R(sec_vlan_cksum_ptype_rss,                                            \
1326           R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)                     \
1327         R(sec_vlan_mark, R_SEC_F | RX_VLAN_F | MARK_F)                         \
1328         R(sec_vlan_mark_rss, R_SEC_F | RX_VLAN_F | MARK_F | RSS_F)             \
1329         R(sec_vlan_mark_ptype, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F)         \
1330         R(sec_vlan_mark_ptype_rss,                                             \
1331           R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)                      \
1332         R(sec_vlan_mark_cksum, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F)         \
1333         R(sec_vlan_mark_cksum_rss,                                             \
1334           R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)                      \
1335         R(sec_vlan_mark_cksum_ptype,                                           \
1336           R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)                    \
1337         R(sec_vlan_mark_cksum_ptype_rss,                                       \
1338           R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1339
1340 #define NIX_RX_FASTPATH_MODES_112_127                                          \
1341         R(sec_vlan_ts, R_SEC_F | RX_VLAN_F | TS_F)                             \
1342         R(sec_vlan_ts_rss, R_SEC_F | RX_VLAN_F | TS_F | RSS_F)                 \
1343         R(sec_vlan_ts_ptype, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F)             \
1344         R(sec_vlan_ts_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
1345         R(sec_vlan_ts_cksum, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F)             \
1346         R(sec_vlan_ts_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
1347         R(sec_vlan_ts_cksum_ptype,                                             \
1348           R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)                      \
1349         R(sec_vlan_ts_cksum_ptype_rss,                                         \
1350           R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)              \
1351         R(sec_vlan_ts_mark, R_SEC_F | RX_VLAN_F | TS_F | MARK_F)               \
1352         R(sec_vlan_ts_mark_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F)   \
1353         R(sec_vlan_ts_mark_ptype,                                              \
1354           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F)                       \
1355         R(sec_vlan_ts_mark_ptype_rss,                                          \
1356           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F)               \
1357         R(sec_vlan_ts_mark_cksum,                                              \
1358           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F)                       \
1359         R(sec_vlan_ts_mark_cksum_rss,                                          \
1360           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F)               \
1361         R(sec_vlan_ts_mark_cksum_ptype,                                        \
1362           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)             \
1363         R(sec_vlan_ts_mark_cksum_ptype_rss,                                    \
1364           R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
1365
1366 #define NIX_RX_FASTPATH_MODES                                                  \
1367         NIX_RX_FASTPATH_MODES_0_15                                             \
1368         NIX_RX_FASTPATH_MODES_16_31                                            \
1369         NIX_RX_FASTPATH_MODES_32_47                                            \
1370         NIX_RX_FASTPATH_MODES_48_63                                            \
1371         NIX_RX_FASTPATH_MODES_64_79                                            \
1372         NIX_RX_FASTPATH_MODES_80_95                                            \
1373         NIX_RX_FASTPATH_MODES_96_111                                           \
1374         NIX_RX_FASTPATH_MODES_112_127
1375
1376 #define R(name, flags)                                                         \
1377         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_##name(          \
1378                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1379         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_mseg_##name(     \
1380                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1381         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_##name(      \
1382                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
1383         uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_mseg_##name( \
1384                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);
1385
1386 NIX_RX_FASTPATH_MODES
1387 #undef R
1388
1389 #define NIX_RX_RECV(fn, flags)                                                 \
1390         uint16_t __rte_noinline __rte_hot fn(                                  \
1391                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
1392         {                                                                      \
1393                 return cn10k_nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags));  \
1394         }
1395
1396 #define NIX_RX_RECV_MSEG(fn, flags) NIX_RX_RECV(fn, flags | NIX_RX_MULTI_SEG_F)
1397
1398 #define NIX_RX_RECV_VEC(fn, flags)                                             \
1399         uint16_t __rte_noinline __rte_hot fn(                                  \
1400                 void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
1401         {                                                                      \
1402                 return cn10k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts,     \
1403                                                   (flags), NULL, NULL, 0);     \
1404         }
1405
1406 #define NIX_RX_RECV_VEC_MSEG(fn, flags)                                        \
1407         NIX_RX_RECV_VEC(fn, flags | NIX_RX_MULTI_SEG_F)
1408
1409 #endif /* __CN10K_RX_H__ */