X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fcnxk%2Fcn9k_rx.h;h=25a4927a33692ec83fbe9a54d0941497584b92c6;hb=fe3833ed2222c369f9b60c0be520dee6e875dcdf;hp=bdedeab3c89b2d8d8f27de3fbf21ce288e2ea617;hpb=c19f95b3201e8781642cd77537d001bb44357026;p=dpdk.git diff --git a/drivers/net/cnxk/cn9k_rx.h b/drivers/net/cnxk/cn9k_rx.h index bdedeab3c8..25a4927a33 100644 --- a/drivers/net/cnxk/cn9k_rx.h +++ b/drivers/net/cnxk/cn9k_rx.h @@ -18,6 +18,7 @@ #define NIX_RX_OFFLOAD_TSTAMP_F BIT(4) #define NIX_RX_OFFLOAD_VLAN_STRIP_F BIT(5) #define NIX_RX_OFFLOAD_SECURITY_F BIT(6) +#define NIX_RX_OFFLOAD_MAX (NIX_RX_OFFLOAD_SECURITY_F << 1) /* Flags to control cqe_to_mbuf conversion function. * Defining it from backwards to denote its been @@ -31,6 +32,9 @@ #define CQE_CAST(x) ((struct nix_cqe_hdr_s *)(x)) #define CQE_SZ(x) ((x) * CNXK_NIX_CQ_ENTRY_SZ) +#define IPSEC_SQ_LO_IDX 4 +#define IPSEC_SQ_HI_IDX 8 + union mbuf_initializer { struct { uint16_t data_off; @@ -100,9 +104,9 @@ nix_update_match_id(const uint16_t match_id, uint64_t ol_flags, * 0 to CNXK_FLOW_ACTION_FLAG_DEFAULT - 2 */ if (likely(match_id)) { - ol_flags |= PKT_RX_FDIR; + ol_flags |= RTE_MBUF_F_RX_FDIR; if (match_id != CNXK_FLOW_ACTION_FLAG_DEFAULT) { - ol_flags |= PKT_RX_FDIR_ID; + ol_flags |= RTE_MBUF_F_RX_FDIR_ID; mbuf->hash.fdir.hi = match_id - 1; } } @@ -148,7 +152,7 @@ nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf, mbuf->next = ((struct rte_mbuf *)*iova_list) - 1; mbuf = mbuf->next; - __mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); mbuf->data_len = sg & 0xFFFF; sg = sg >> 16; @@ -166,6 +170,94 @@ nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf, mbuf->next = NULL; } +static inline int +ipsec_antireplay_check(struct roc_onf_ipsec_inb_sa *sa, + struct cn9k_inb_priv_data *priv, uintptr_t data, + uint32_t win_sz) +{ + struct cnxk_on_ipsec_ar *ar = &priv->ar; + uint64_t seq_in_sa; + uint32_t seqh = 0; + uint32_t seql; + uint64_t seq; + uint8_t esn; + int rc; + + esn = sa->ctl.esn_en; + seql = rte_be_to_cpu_32(*((uint32_t *)(data + IPSEC_SQ_LO_IDX))); + + if (!esn) { + seq = (uint64_t)seql; + } else { + seqh = rte_be_to_cpu_32(*((uint32_t *)(data + + IPSEC_SQ_HI_IDX))); + seq = ((uint64_t)seqh << 32) | seql; + } + + if (unlikely(seq == 0)) + return -1; + + rte_spinlock_lock(&ar->lock); + rc = cnxk_on_anti_replay_check(seq, ar, win_sz); + if (esn && !rc) { + seq_in_sa = ((uint64_t)rte_be_to_cpu_32(sa->esn_hi) << 32) | + rte_be_to_cpu_32(sa->esn_low); + if (seq > seq_in_sa) { + sa->esn_low = rte_cpu_to_be_32(seql); + sa->esn_hi = rte_cpu_to_be_32(seqh); + } + } + rte_spinlock_unlock(&ar->lock); + + return rc; +} + +static inline uint64_t +nix_rx_sec_mbuf_err_update(const union nix_rx_parse_u *rx, uint16_t res, + uint64_t *rearm_val, uint16_t *len) +{ + uint8_t uc_cc = res >> 8; + uint8_t cc = res & 0xFF; + uint64_t data_off; + uint64_t ol_flags; + uint16_t m_len; + + if (unlikely(cc != CPT_COMP_GOOD)) + return RTE_MBUF_F_RX_SEC_OFFLOAD | + RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; + + data_off = *rearm_val & (BIT_ULL(16) - 1); + m_len = rx->cn9k.pkt_lenm1 + 1; + + switch (uc_cc) { + case ROC_IE_ON_UCC_IP_PAYLOAD_TYPE_ERR: + case ROC_IE_ON_UCC_AUTH_ERR: + case ROC_IE_ON_UCC_PADDING_INVALID: + /* Adjust data offset to start at copied L2 */ + data_off += ROC_ONF_IPSEC_INB_SPI_SEQ_SZ + + ROC_ONF_IPSEC_INB_MAX_L2_SZ; + ol_flags = RTE_MBUF_F_RX_SEC_OFFLOAD | + RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; + break; + case ROC_IE_ON_UCC_CTX_INVALID: + case ROC_IE_ON_UCC_SPI_MISMATCH: + case ROC_IE_ON_UCC_SA_MISMATCH: + /* Return as normal packet */ + ol_flags = 0; + break; + default: + /* Return as error packet after updating packet lengths */ + ol_flags = RTE_MBUF_F_RX_SEC_OFFLOAD | + RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; + break; + } + + *len = m_len; + *rearm_val = *rearm_val & ~(BIT_ULL(16) - 1); + *rearm_val |= data_off; + return ol_flags; +} + static __rte_always_inline uint64_t nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, uintptr_t sa_base, uint64_t *rearm_val, uint16_t *len) @@ -178,8 +270,8 @@ nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, uint8_t lcptr = rx->lcptr; struct rte_ipv4_hdr *ipv4; uint16_t data_off, res; + uint32_t spi, win_sz; uint32_t spi_mask; - uint32_t spi; uintptr_t data; __uint128_t dw; uint8_t sa_w; @@ -191,8 +283,8 @@ nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, rte_prefetch0((void *)data); - if (unlikely(res != (CPT_COMP_GOOD | ROC_IE_ONF_UCC_SUCCESS << 8))) - return PKT_RX_SEC_OFFLOAD | PKT_RX_SEC_OFFLOAD_FAILED; + if (unlikely(res != (CPT_COMP_GOOD | ROC_IE_ON_UCC_SUCCESS << 8))) + return nix_rx_sec_mbuf_err_update(rx, res, rearm_val, len); data += lcptr; /* 20 bits of tag would have the SPI */ @@ -209,6 +301,13 @@ nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, dw = *(__uint128_t *)sa_priv; *rte_security_dynfield(m) = (uint64_t)dw; + /* Check if anti-replay is enabled */ + win_sz = (uint32_t)(dw >> 64); + if (win_sz) { + if (ipsec_antireplay_check(sa, sa_priv, data, win_sz) < 0) + return RTE_MBUF_F_RX_SEC_OFFLOAD | RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; + } + /* Get total length from IPv4 header. We can assume only IPv4 */ ipv4 = (struct rte_ipv4_hdr *)(data + ROC_ONF_IPSEC_INB_SPI_SEQ_SZ + ROC_ONF_IPSEC_INB_MAX_L2_SZ); @@ -220,7 +319,7 @@ nix_rx_sec_mbuf_update(const struct nix_cqe_hdr_s *cq, struct rte_mbuf *m, *rearm_val |= data_off; *len = rte_be_to_cpu_16(ipv4->total_length) + lcptr; - return PKT_RX_SEC_OFFLOAD; + return RTE_MBUF_F_RX_SEC_OFFLOAD; } static __rte_always_inline void @@ -236,7 +335,7 @@ cn9k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, uint64_t ol_flags = 0; /* Mark mempool obj as "get" as it is alloc'ed by NIX */ - __mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1); if (flag & NIX_RX_OFFLOAD_PTYPE_F) packet_type = nix_ptype_get(lookup_mem, w1); @@ -267,7 +366,7 @@ cn9k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, if (flag & NIX_RX_OFFLOAD_RSS_F) { mbuf->hash.rss = tag; - ol_flags |= PKT_RX_RSS_HASH; + ol_flags |= RTE_MBUF_F_RX_RSS_HASH; } if (flag & NIX_RX_OFFLOAD_CHECKSUM_F) @@ -276,11 +375,11 @@ cn9k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, skip_parse: if (flag & NIX_RX_OFFLOAD_VLAN_STRIP_F) { if (rx->cn9k.vtag0_gone) { - ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; + ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; mbuf->vlan_tci = rx->cn9k.vtag0_tci; } if (rx->cn9k.vtag1_gone) { - ol_flags |= PKT_RX_QINQ | PKT_RX_QINQ_STRIPPED; + ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED; mbuf->vlan_tci_outer = rx->cn9k.vtag1_tci; } } @@ -289,14 +388,19 @@ skip_parse: ol_flags = nix_update_match_id(rx->cn9k.match_id, ol_flags, mbuf); + mbuf->ol_flags = ol_flags; + *(uint64_t *)(&mbuf->rearm_data) = val; mbuf->pkt_len = len; mbuf->data_len = len; - *(uint64_t *)(&mbuf->rearm_data) = val; - - mbuf->ol_flags = ol_flags; if (flag & NIX_RX_MULTI_SEG_F) - nix_cqe_xtract_mseg(rx, mbuf, val, flag); + /* + * For multi segment packets, mbuf length correction according + * to Rx timestamp length will be handled later during + * timestamp data process. + * Hence, flag argument is not required. + */ + nix_cqe_xtract_mseg(rx, mbuf, val, 0); else mbuf->next = NULL; } @@ -331,6 +435,34 @@ nix_rx_nb_pkts(struct cn9k_eth_rxq *rxq, const uint64_t wdata, return RTE_MIN(pkts, available); } +static __rte_always_inline void +cn9k_nix_mbuf_to_tstamp(struct rte_mbuf *mbuf, + struct cnxk_timesync_info *tstamp, + const uint8_t ts_enable, uint64_t *tstamp_ptr) +{ + if (ts_enable) { + mbuf->pkt_len -= CNXK_NIX_TIMESYNC_RX_OFFSET; + mbuf->data_len -= CNXK_NIX_TIMESYNC_RX_OFFSET; + + /* Reading the rx timestamp inserted by CGX, viz at + * starting of the packet data. + */ + *cnxk_nix_timestamp_dynfield(mbuf, tstamp) = + rte_be_to_cpu_64(*tstamp_ptr); + /* RTE_MBUF_F_RX_IEEE1588_TMST flag needs to be set only in case + * PTP packets are received. + */ + if (mbuf->packet_type == RTE_PTYPE_L2_ETHER_TIMESYNC) { + tstamp->rx_tstamp = + *cnxk_nix_timestamp_dynfield(mbuf, tstamp); + tstamp->rx_ready = 1; + mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP | + RTE_MBUF_F_RX_IEEE1588_TMST | + tstamp->rx_tstamp_dynflag; + } + } +} + static __rte_always_inline uint16_t cn9k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts, const uint16_t flags) @@ -359,9 +491,8 @@ cn9k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts, cn9k_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init, flags); - cnxk_nix_mbuf_to_tstamp(mbuf, rxq->tstamp, + cn9k_nix_mbuf_to_tstamp(mbuf, rxq->tstamp, (flags & NIX_RX_OFFLOAD_TSTAMP_F), - (flags & NIX_RX_MULTI_SEG_F), (uint64_t *)((uint8_t *)mbuf + data_off)); rx_pkts[packets++] = mbuf; @@ -385,7 +516,7 @@ static __rte_always_inline uint64_t nix_vlan_update(const uint64_t w2, uint64_t ol_flags, uint8x16_t *f) { if (w2 & BIT_ULL(21) /* vtag0_gone */) { - ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; + ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; *f = vsetq_lane_u16((uint16_t)(w2 >> 32), *f, 5); } @@ -396,7 +527,7 @@ static __rte_always_inline uint64_t nix_qinq_update(const uint64_t w2, uint64_t ol_flags, struct rte_mbuf *mbuf) { if (w2 & BIT_ULL(23) /* vtag1_gone */) { - ol_flags |= PKT_RX_QINQ | PKT_RX_QINQ_STRIPPED; + ol_flags |= RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_RX_QINQ_STRIPPED; mbuf->vlan_tci_outer = (uint16_t)(w2 >> 48); } @@ -497,10 +628,10 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts, f1 = vsetq_lane_u32(cq1_w0, f1, 3); f2 = vsetq_lane_u32(cq2_w0, f2, 3); f3 = vsetq_lane_u32(cq3_w0, f3, 3); - ol_flags0 = PKT_RX_RSS_HASH; - ol_flags1 = PKT_RX_RSS_HASH; - ol_flags2 = PKT_RX_RSS_HASH; - ol_flags3 = PKT_RX_RSS_HASH; + ol_flags0 = RTE_MBUF_F_RX_RSS_HASH; + ol_flags1 = RTE_MBUF_F_RX_RSS_HASH; + ol_flags2 = RTE_MBUF_F_RX_RSS_HASH; + ol_flags3 = RTE_MBUF_F_RX_RSS_HASH; } else { ol_flags0 = 0; ol_flags1 = 0; @@ -573,8 +704,8 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts, RTE_PTYPE_L2_ETHER_TIMESYNC, RTE_PTYPE_L2_ETHER_TIMESYNC, RTE_PTYPE_L2_ETHER_TIMESYNC}; - const uint64_t ts_olf = PKT_RX_IEEE1588_PTP | - PKT_RX_IEEE1588_TMST | + const uint64_t ts_olf = RTE_MBUF_F_RX_IEEE1588_PTP | + RTE_MBUF_F_RX_IEEE1588_TMST | rxq->tstamp->rx_tstamp_dynflag; const uint32x4_t and_mask = {0x1, 0x2, 0x4, 0x8}; uint64x2_t ts01, ts23, mask; @@ -670,10 +801,6 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts, vst1q_u64((uint64_t *)mbuf2->rearm_data, rearm2); vst1q_u64((uint64_t *)mbuf3->rearm_data, rearm3); - /* Store the mbufs to rx_pkts */ - vst1q_u64((uint64_t *)&rx_pkts[packets], mbuf01); - vst1q_u64((uint64_t *)&rx_pkts[packets + 2], mbuf23); - if (flags & NIX_RX_MULTI_SEG_F) { /* Multi segment is enable build mseg list for * individual mbufs in scalar mode. @@ -698,6 +825,10 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts, mbuf3->next = NULL; } + /* Store the mbufs to rx_pkts */ + vst1q_u64((uint64_t *)&rx_pkts[packets], mbuf01); + vst1q_u64((uint64_t *)&rx_pkts[packets + 2], mbuf23); + /* Prefetch mbufs */ roc_prefetch_store_keep(mbuf0); roc_prefetch_store_keep(mbuf1); @@ -705,10 +836,10 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts, roc_prefetch_store_keep(mbuf3); /* Mark mempool obj as "get" as it is alloc'ed by NIX */ - __mempool_check_cookies(mbuf0->pool, (void **)&mbuf0, 1, 1); - __mempool_check_cookies(mbuf1->pool, (void **)&mbuf1, 1, 1); - __mempool_check_cookies(mbuf2->pool, (void **)&mbuf2, 1, 1); - __mempool_check_cookies(mbuf3->pool, (void **)&mbuf3, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1); /* Advance head pointer and packets */ head += NIX_DESCS_PER_LOOP; @@ -755,278 +886,211 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts, #define R_SEC_F NIX_RX_OFFLOAD_SECURITY_F /* [R_SEC_F] [RX_VLAN_F] [TS] [MARK] [CKSUM] [PTYPE] [RSS] */ -#define NIX_RX_FASTPATH_MODES \ -R(no_offload, 0, 0, 0, 0, 0, 0, 0, \ - NIX_RX_OFFLOAD_NONE) \ -R(rss, 0, 0, 0, 0, 0, 0, 1, \ - RSS_F) \ -R(ptype, 0, 0, 0, 0, 0, 1, 0, \ - PTYPE_F) \ -R(ptype_rss, 0, 0, 0, 0, 0, 1, 1, \ - PTYPE_F | RSS_F) \ -R(cksum, 0, 0, 0, 0, 1, 0, 0, \ - CKSUM_F) \ -R(cksum_rss, 0, 0, 0, 0, 1, 0, 1, \ - CKSUM_F | RSS_F) \ -R(cksum_ptype, 0, 0, 0, 0, 1, 1, 0, \ - CKSUM_F | PTYPE_F) \ -R(cksum_ptype_rss, 0, 0, 0, 0, 1, 1, 1, \ - CKSUM_F | PTYPE_F | RSS_F) \ -R(mark, 0, 0, 0, 1, 0, 0, 0, \ - MARK_F) \ -R(mark_rss, 0, 0, 0, 1, 0, 0, 1, \ - MARK_F | RSS_F) \ -R(mark_ptype, 0, 0, 0, 1, 0, 1, 0, \ - MARK_F | PTYPE_F) \ -R(mark_ptype_rss, 0, 0, 0, 1, 0, 1, 1, \ - MARK_F | PTYPE_F | RSS_F) \ -R(mark_cksum, 0, 0, 0, 1, 1, 0, 0, \ - MARK_F | CKSUM_F) \ -R(mark_cksum_rss, 0, 0, 0, 1, 1, 0, 1, \ - MARK_F | CKSUM_F | RSS_F) \ -R(mark_cksum_ptype, 0, 0, 0, 1, 1, 1, 0, \ - MARK_F | CKSUM_F | PTYPE_F) \ -R(mark_cksum_ptype_rss, 0, 0, 0, 1, 1, 1, 1, \ - MARK_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(ts, 0, 0, 1, 0, 0, 0, 0, \ - TS_F) \ -R(ts_rss, 0, 0, 1, 0, 0, 0, 1, \ - TS_F | RSS_F) \ -R(ts_ptype, 0, 0, 1, 0, 0, 1, 0, \ - TS_F | PTYPE_F) \ -R(ts_ptype_rss, 0, 0, 1, 0, 0, 1, 1, \ - TS_F | PTYPE_F | RSS_F) \ -R(ts_cksum, 0, 0, 1, 0, 1, 0, 0, \ - TS_F | CKSUM_F) \ -R(ts_cksum_rss, 0, 0, 1, 0, 1, 0, 1, \ - TS_F | CKSUM_F | RSS_F) \ -R(ts_cksum_ptype, 0, 0, 1, 0, 1, 1, 0, \ - TS_F | CKSUM_F | PTYPE_F) \ -R(ts_cksum_ptype_rss, 0, 0, 1, 0, 1, 1, 1, \ - TS_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(ts_mark, 0, 0, 1, 1, 0, 0, 0, \ - TS_F | MARK_F) \ -R(ts_mark_rss, 0, 0, 1, 1, 0, 0, 1, \ - TS_F | MARK_F | RSS_F) \ -R(ts_mark_ptype, 0, 0, 1, 1, 0, 1, 0, \ - TS_F | MARK_F | PTYPE_F) \ -R(ts_mark_ptype_rss, 0, 0, 1, 1, 0, 1, 1, \ - TS_F | MARK_F | PTYPE_F | RSS_F) \ -R(ts_mark_cksum, 0, 0, 1, 1, 1, 0, 0, \ - TS_F | MARK_F | CKSUM_F) \ -R(ts_mark_cksum_rss, 0, 0, 1, 1, 1, 0, 1, \ - TS_F | MARK_F | CKSUM_F | RSS_F) \ -R(ts_mark_cksum_ptype, 0, 0, 1, 1, 1, 1, 0, \ - TS_F | MARK_F | CKSUM_F | PTYPE_F) \ -R(ts_mark_cksum_ptype_rss, 0, 0, 1, 1, 1, 1, 1, \ - TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(vlan, 0, 1, 0, 0, 0, 0, 0, \ - RX_VLAN_F) \ -R(vlan_rss, 0, 1, 0, 0, 0, 0, 1, \ - RX_VLAN_F | RSS_F) \ -R(vlan_ptype, 0, 1, 0, 0, 0, 1, 0, \ - RX_VLAN_F | PTYPE_F) \ -R(vlan_ptype_rss, 0, 1, 0, 0, 0, 1, 1, \ - RX_VLAN_F | PTYPE_F | RSS_F) \ -R(vlan_cksum, 0, 1, 0, 0, 1, 0, 0, \ - RX_VLAN_F | CKSUM_F) \ -R(vlan_cksum_rss, 0, 1, 0, 0, 1, 0, 1, \ - RX_VLAN_F | CKSUM_F | RSS_F) \ -R(vlan_cksum_ptype, 0, 1, 0, 0, 1, 1, 0, \ - RX_VLAN_F | CKSUM_F | PTYPE_F) \ -R(vlan_cksum_ptype_rss, 0, 1, 0, 0, 1, 1, 1, \ - RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(vlan_mark, 0, 1, 0, 1, 0, 0, 0, \ - RX_VLAN_F | MARK_F) \ -R(vlan_mark_rss, 0, 1, 0, 1, 0, 0, 1, \ - RX_VLAN_F | MARK_F | RSS_F) \ -R(vlan_mark_ptype, 0, 1, 0, 1, 0, 1, 0, \ - RX_VLAN_F | MARK_F | PTYPE_F) \ -R(vlan_mark_ptype_rss, 0, 1, 0, 1, 0, 1, 1, \ - RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \ -R(vlan_mark_cksum, 0, 1, 0, 1, 1, 0, 0, \ - RX_VLAN_F | MARK_F | CKSUM_F) \ -R(vlan_mark_cksum_rss, 0, 1, 0, 1, 1, 0, 1, \ - RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \ -R(vlan_mark_cksum_ptype, 0, 1, 0, 1, 1, 1, 0, \ - RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \ -R(vlan_mark_cksum_ptype_rss, 0, 1, 0, 1, 1, 1, 1, \ - RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(vlan_ts, 0, 1, 1, 0, 0, 0, 0, \ - RX_VLAN_F | TS_F) \ -R(vlan_ts_rss, 0, 1, 1, 0, 0, 0, 1, \ - RX_VLAN_F | TS_F | RSS_F) \ -R(vlan_ts_ptype, 0, 1, 1, 0, 0, 1, 0, \ - RX_VLAN_F | TS_F | PTYPE_F) \ -R(vlan_ts_ptype_rss, 0, 1, 1, 0, 0, 1, 1, \ - RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \ -R(vlan_ts_cksum, 0, 1, 1, 0, 1, 0, 0, \ - RX_VLAN_F | TS_F | CKSUM_F) \ -R(vlan_ts_cksum_rss, 0, 1, 1, 0, 1, 0, 1, \ - RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \ -R(vlan_ts_cksum_ptype, 0, 1, 1, 0, 1, 1, 0, \ - RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \ -R(vlan_ts_cksum_ptype_rss, 0, 1, 1, 0, 1, 1, 1, \ - RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(vlan_ts_mark, 0, 1, 1, 1, 0, 0, 0, \ - RX_VLAN_F | TS_F | MARK_F) \ -R(vlan_ts_mark_rss, 0, 1, 1, 1, 0, 0, 1, \ - RX_VLAN_F | TS_F | MARK_F | RSS_F) \ -R(vlan_ts_mark_ptype, 0, 1, 1, 1, 0, 1, 0, \ - RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \ -R(vlan_ts_mark_ptype_rss, 0, 1, 1, 1, 0, 1, 1, \ - RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ -R(vlan_ts_mark_cksum, 0, 1, 1, 1, 1, 0, 0, \ - RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \ -R(vlan_ts_mark_cksum_rss, 0, 1, 1, 1, 1, 0, 1, \ - RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ -R(vlan_ts_mark_cksum_ptype, 0, 1, 1, 1, 1, 1, 0, \ - RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ -R(vlan_ts_mark_cksum_ptype_rss, 0, 1, 1, 1, 1, 1, 1, \ - RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec, 1, 0, 0, 0, 0, 0, 0, \ - R_SEC_F) \ -R(sec_rss, 1, 0, 0, 0, 0, 0, 1, \ - RSS_F) \ -R(sec_ptype, 1, 0, 0, 0, 0, 1, 0, \ - R_SEC_F | PTYPE_F) \ -R(sec_ptype_rss, 1, 0, 0, 0, 0, 1, 1, \ - R_SEC_F | PTYPE_F | RSS_F) \ -R(sec_cksum, 1, 0, 0, 0, 1, 0, 0, \ - R_SEC_F | CKSUM_F) \ -R(sec_cksum_rss, 1, 0, 0, 0, 1, 0, 1, \ - R_SEC_F | CKSUM_F | RSS_F) \ -R(sec_cksum_ptype, 1, 0, 0, 0, 1, 1, 0, \ - R_SEC_F | CKSUM_F | PTYPE_F) \ -R(sec_cksum_ptype_rss, 1, 0, 0, 0, 1, 1, 1, \ - R_SEC_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec_mark, 1, 0, 0, 1, 0, 0, 0, \ - R_SEC_F | MARK_F) \ -R(sec_mark_rss, 1, 0, 0, 1, 0, 0, 1, \ - R_SEC_F | MARK_F | RSS_F) \ -R(sec_mark_ptype, 1, 0, 0, 1, 0, 1, 0, \ - R_SEC_F | MARK_F | PTYPE_F) \ -R(sec_mark_ptype_rss, 1, 0, 0, 1, 0, 1, 1, \ - R_SEC_F | MARK_F | PTYPE_F | RSS_F) \ -R(sec_mark_cksum, 1, 0, 0, 1, 1, 0, 0, \ - R_SEC_F | MARK_F | CKSUM_F) \ -R(sec_mark_cksum_rss, 1, 0, 0, 1, 1, 0, 1, \ - R_SEC_F | MARK_F | CKSUM_F | RSS_F) \ -R(sec_mark_cksum_ptype, 1, 0, 0, 1, 1, 1, 0, \ - R_SEC_F | MARK_F | CKSUM_F | PTYPE_F) \ -R(sec_mark_cksum_ptype_rss, 1, 0, 0, 1, 1, 1, 1, \ - R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec_ts, 1, 0, 1, 0, 0, 0, 0, \ - R_SEC_F | TS_F) \ -R(sec_ts_rss, 1, 0, 1, 0, 0, 0, 1, \ - R_SEC_F | TS_F | RSS_F) \ -R(sec_ts_ptype, 1, 0, 1, 0, 0, 1, 0, \ - R_SEC_F | TS_F | PTYPE_F) \ -R(sec_ts_ptype_rss, 1, 0, 1, 0, 0, 1, 1, \ - R_SEC_F | TS_F | PTYPE_F | RSS_F) \ -R(sec_ts_cksum, 1, 0, 1, 0, 1, 0, 0, \ - R_SEC_F | TS_F | CKSUM_F) \ -R(sec_ts_cksum_rss, 1, 0, 1, 0, 1, 0, 1, \ - R_SEC_F | TS_F | CKSUM_F | RSS_F) \ -R(sec_ts_cksum_ptype, 1, 0, 1, 0, 1, 1, 0, \ - R_SEC_F | TS_F | CKSUM_F | PTYPE_F) \ -R(sec_ts_cksum_ptype_rss, 1, 0, 1, 0, 1, 1, 1, \ - R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec_ts_mark, 1, 0, 1, 1, 0, 0, 0, \ - R_SEC_F | TS_F | MARK_F) \ -R(sec_ts_mark_rss, 1, 0, 1, 1, 0, 0, 1, \ - R_SEC_F | TS_F | MARK_F | RSS_F) \ -R(sec_ts_mark_ptype, 1, 0, 1, 1, 0, 1, 0, \ - R_SEC_F | TS_F | MARK_F | PTYPE_F) \ -R(sec_ts_mark_ptype_rss, 1, 0, 1, 1, 0, 1, 1, \ - R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ -R(sec_ts_mark_cksum, 1, 0, 1, 1, 1, 0, 0, \ - R_SEC_F | TS_F | MARK_F | CKSUM_F) \ -R(sec_ts_mark_cksum_rss, 1, 0, 1, 1, 1, 0, 1, \ - R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ -R(sec_ts_mark_cksum_ptype, 1, 0, 1, 1, 1, 1, 0, \ - R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ -R(sec_ts_mark_cksum_ptype_rss, 1, 0, 1, 1, 1, 1, 1, \ - R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec_vlan, 1, 1, 0, 0, 0, 0, 0, \ - R_SEC_F | RX_VLAN_F) \ -R(sec_vlan_rss, 1, 1, 0, 0, 0, 0, 1, \ - R_SEC_F | RX_VLAN_F | RSS_F) \ -R(sec_vlan_ptype, 1, 1, 0, 0, 0, 1, 0, \ - R_SEC_F | RX_VLAN_F | PTYPE_F) \ -R(sec_vlan_ptype_rss, 1, 1, 0, 0, 0, 1, 1, \ - R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F) \ -R(sec_vlan_cksum, 1, 1, 0, 0, 1, 0, 0, \ - R_SEC_F | RX_VLAN_F | CKSUM_F) \ -R(sec_vlan_cksum_rss, 1, 1, 0, 0, 1, 0, 1, \ - R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F) \ -R(sec_vlan_cksum_ptype, 1, 1, 0, 0, 1, 1, 0, \ - R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F) \ -R(sec_vlan_cksum_ptype_rss, 1, 1, 0, 0, 1, 1, 1, \ - R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec_vlan_mark, 1, 1, 0, 1, 0, 0, 0, \ - R_SEC_F | RX_VLAN_F | MARK_F) \ -R(sec_vlan_mark_rss, 1, 1, 0, 1, 0, 0, 1, \ - R_SEC_F | RX_VLAN_F | MARK_F | RSS_F) \ -R(sec_vlan_mark_ptype, 1, 1, 0, 1, 0, 1, 0, \ - R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F) \ -R(sec_vlan_mark_ptype_rss, 1, 1, 0, 1, 0, 1, 1, \ - R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \ -R(sec_vlan_mark_cksum, 1, 1, 0, 1, 1, 0, 0, \ - R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F) \ -R(sec_vlan_mark_cksum_rss, 1, 1, 0, 1, 1, 0, 1, \ - R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \ -R(sec_vlan_mark_cksum_ptype, 1, 1, 0, 1, 1, 1, 0, \ - R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \ -R(sec_vlan_mark_cksum_ptype_rss, 1, 1, 0, 1, 1, 1, 1, \ - R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec_vlan_ts, 1, 1, 1, 0, 0, 0, 0, \ - R_SEC_F | RX_VLAN_F | TS_F) \ -R(sec_vlan_ts_rss, 1, 1, 1, 0, 0, 0, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | RSS_F) \ -R(sec_vlan_ts_ptype, 1, 1, 1, 0, 0, 1, 0, \ - R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F) \ -R(sec_vlan_ts_ptype_rss, 1, 1, 1, 0, 0, 1, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \ -R(sec_vlan_ts_cksum, 1, 1, 1, 0, 1, 0, 0, \ - R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F) \ -R(sec_vlan_ts_cksum_rss, 1, 1, 1, 0, 1, 0, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \ -R(sec_vlan_ts_cksum_ptype, 1, 1, 1, 0, 1, 1, 0, \ - R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \ -R(sec_vlan_ts_cksum_ptype_rss, 1, 1, 1, 0, 1, 1, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ -R(sec_vlan_ts_mark, 1, 1, 1, 1, 0, 0, 0, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F) \ -R(sec_vlan_ts_mark_rss, 1, 1, 1, 1, 0, 0, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F) \ -R(sec_vlan_ts_mark_ptype, 1, 1, 1, 1, 0, 1, 0, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \ -R(sec_vlan_ts_mark_ptype_rss, 1, 1, 1, 1, 0, 1, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ -R(sec_vlan_ts_mark_cksum, 1, 1, 1, 1, 1, 0, 0, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \ -R(sec_vlan_ts_mark_cksum_rss, 1, 1, 1, 1, 1, 0, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ -R(sec_vlan_ts_mark_cksum_ptype, 1, 1, 1, 1, 1, 1, 0, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ -R(sec_vlan_ts_mark_cksum_ptype_rss, 1, 1, 1, 1, 1, 1, 1, \ - R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) - -#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \ +#define NIX_RX_FASTPATH_MODES_0_15 \ + R(no_offload, NIX_RX_OFFLOAD_NONE) \ + R(rss, RSS_F) \ + R(ptype, PTYPE_F) \ + R(ptype_rss, PTYPE_F | RSS_F) \ + R(cksum, CKSUM_F) \ + R(cksum_rss, CKSUM_F | RSS_F) \ + R(cksum_ptype, CKSUM_F | PTYPE_F) \ + R(cksum_ptype_rss, CKSUM_F | PTYPE_F | RSS_F) \ + R(mark, MARK_F) \ + R(mark_rss, MARK_F | RSS_F) \ + R(mark_ptype, MARK_F | PTYPE_F) \ + R(mark_ptype_rss, MARK_F | PTYPE_F | RSS_F) \ + R(mark_cksum, MARK_F | CKSUM_F) \ + R(mark_cksum_rss, MARK_F | CKSUM_F | RSS_F) \ + R(mark_cksum_ptype, MARK_F | CKSUM_F | PTYPE_F) \ + R(mark_cksum_ptype_rss, MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES_16_31 \ + R(ts, TS_F) \ + R(ts_rss, TS_F | RSS_F) \ + R(ts_ptype, TS_F | PTYPE_F) \ + R(ts_ptype_rss, TS_F | PTYPE_F | RSS_F) \ + R(ts_cksum, TS_F | CKSUM_F) \ + R(ts_cksum_rss, TS_F | CKSUM_F | RSS_F) \ + R(ts_cksum_ptype, TS_F | CKSUM_F | PTYPE_F) \ + R(ts_cksum_ptype_rss, TS_F | CKSUM_F | PTYPE_F | RSS_F) \ + R(ts_mark, TS_F | MARK_F) \ + R(ts_mark_rss, TS_F | MARK_F | RSS_F) \ + R(ts_mark_ptype, TS_F | MARK_F | PTYPE_F) \ + R(ts_mark_ptype_rss, TS_F | MARK_F | PTYPE_F | RSS_F) \ + R(ts_mark_cksum, TS_F | MARK_F | CKSUM_F) \ + R(ts_mark_cksum_rss, TS_F | MARK_F | CKSUM_F | RSS_F) \ + R(ts_mark_cksum_ptype, TS_F | MARK_F | CKSUM_F | PTYPE_F) \ + R(ts_mark_cksum_ptype_rss, TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES_32_47 \ + R(vlan, RX_VLAN_F) \ + R(vlan_rss, RX_VLAN_F | RSS_F) \ + R(vlan_ptype, RX_VLAN_F | PTYPE_F) \ + R(vlan_ptype_rss, RX_VLAN_F | PTYPE_F | RSS_F) \ + R(vlan_cksum, RX_VLAN_F | CKSUM_F) \ + R(vlan_cksum_rss, RX_VLAN_F | CKSUM_F | RSS_F) \ + R(vlan_cksum_ptype, RX_VLAN_F | CKSUM_F | PTYPE_F) \ + R(vlan_cksum_ptype_rss, RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \ + R(vlan_mark, RX_VLAN_F | MARK_F) \ + R(vlan_mark_rss, RX_VLAN_F | MARK_F | RSS_F) \ + R(vlan_mark_ptype, RX_VLAN_F | MARK_F | PTYPE_F) \ + R(vlan_mark_ptype_rss, RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \ + R(vlan_mark_cksum, RX_VLAN_F | MARK_F | CKSUM_F) \ + R(vlan_mark_cksum_rss, RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \ + R(vlan_mark_cksum_ptype, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \ + R(vlan_mark_cksum_ptype_rss, \ + RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES_48_63 \ + R(vlan_ts, RX_VLAN_F | TS_F) \ + R(vlan_ts_rss, RX_VLAN_F | TS_F | RSS_F) \ + R(vlan_ts_ptype, RX_VLAN_F | TS_F | PTYPE_F) \ + R(vlan_ts_ptype_rss, RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \ + R(vlan_ts_cksum, RX_VLAN_F | TS_F | CKSUM_F) \ + R(vlan_ts_cksum_rss, RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \ + R(vlan_ts_cksum_ptype, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \ + R(vlan_ts_cksum_ptype_rss, \ + RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ + R(vlan_ts_mark, RX_VLAN_F | TS_F | MARK_F) \ + R(vlan_ts_mark_rss, RX_VLAN_F | TS_F | MARK_F | RSS_F) \ + R(vlan_ts_mark_ptype, RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \ + R(vlan_ts_mark_ptype_rss, RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ + R(vlan_ts_mark_cksum, RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \ + R(vlan_ts_mark_cksum_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ + R(vlan_ts_mark_cksum_ptype, \ + RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ + R(vlan_ts_mark_cksum_ptype_rss, \ + RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES_64_79 \ + R(sec, R_SEC_F) \ + R(sec_rss, R_SEC_F | RSS_F) \ + R(sec_ptype, R_SEC_F | PTYPE_F) \ + R(sec_ptype_rss, R_SEC_F | PTYPE_F | RSS_F) \ + R(sec_cksum, R_SEC_F | CKSUM_F) \ + R(sec_cksum_rss, R_SEC_F | CKSUM_F | RSS_F) \ + R(sec_cksum_ptype, R_SEC_F | CKSUM_F | PTYPE_F) \ + R(sec_cksum_ptype_rss, R_SEC_F | CKSUM_F | PTYPE_F | RSS_F) \ + R(sec_mark, R_SEC_F | MARK_F) \ + R(sec_mark_rss, R_SEC_F | MARK_F | RSS_F) \ + R(sec_mark_ptype, R_SEC_F | MARK_F | PTYPE_F) \ + R(sec_mark_ptype_rss, R_SEC_F | MARK_F | PTYPE_F | RSS_F) \ + R(sec_mark_cksum, R_SEC_F | MARK_F | CKSUM_F) \ + R(sec_mark_cksum_rss, R_SEC_F | MARK_F | CKSUM_F | RSS_F) \ + R(sec_mark_cksum_ptype, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F) \ + R(sec_mark_cksum_ptype_rss, \ + R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES_80_95 \ + R(sec_ts, R_SEC_F | TS_F) \ + R(sec_ts_rss, R_SEC_F | TS_F | RSS_F) \ + R(sec_ts_ptype, R_SEC_F | TS_F | PTYPE_F) \ + R(sec_ts_ptype_rss, R_SEC_F | TS_F | PTYPE_F | RSS_F) \ + R(sec_ts_cksum, R_SEC_F | TS_F | CKSUM_F) \ + R(sec_ts_cksum_rss, R_SEC_F | TS_F | CKSUM_F | RSS_F) \ + R(sec_ts_cksum_ptype, R_SEC_F | TS_F | CKSUM_F | PTYPE_F) \ + R(sec_ts_cksum_ptype_rss, R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ + R(sec_ts_mark, R_SEC_F | TS_F | MARK_F) \ + R(sec_ts_mark_rss, R_SEC_F | TS_F | MARK_F | RSS_F) \ + R(sec_ts_mark_ptype, R_SEC_F | TS_F | MARK_F | PTYPE_F) \ + R(sec_ts_mark_ptype_rss, R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ + R(sec_ts_mark_cksum, R_SEC_F | TS_F | MARK_F | CKSUM_F) \ + R(sec_ts_mark_cksum_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ + R(sec_ts_mark_cksum_ptype, \ + R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ + R(sec_ts_mark_cksum_ptype_rss, \ + R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES_96_111 \ + R(sec_vlan, R_SEC_F | RX_VLAN_F) \ + R(sec_vlan_rss, R_SEC_F | RX_VLAN_F | RSS_F) \ + R(sec_vlan_ptype, R_SEC_F | RX_VLAN_F | PTYPE_F) \ + R(sec_vlan_ptype_rss, R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F) \ + R(sec_vlan_cksum, R_SEC_F | RX_VLAN_F | CKSUM_F) \ + R(sec_vlan_cksum_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F) \ + R(sec_vlan_cksum_ptype, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F) \ + R(sec_vlan_cksum_ptype_rss, \ + R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \ + R(sec_vlan_mark, R_SEC_F | RX_VLAN_F | MARK_F) \ + R(sec_vlan_mark_rss, R_SEC_F | RX_VLAN_F | MARK_F | RSS_F) \ + R(sec_vlan_mark_ptype, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F) \ + R(sec_vlan_mark_ptype_rss, \ + R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \ + R(sec_vlan_mark_cksum, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F) \ + R(sec_vlan_mark_cksum_rss, \ + R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \ + R(sec_vlan_mark_cksum_ptype, \ + R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \ + R(sec_vlan_mark_cksum_ptype_rss, \ + R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES_112_127 \ + R(sec_vlan_ts, R_SEC_F | RX_VLAN_F | TS_F) \ + R(sec_vlan_ts_rss, R_SEC_F | RX_VLAN_F | TS_F | RSS_F) \ + R(sec_vlan_ts_ptype, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F) \ + R(sec_vlan_ts_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \ + R(sec_vlan_ts_cksum, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F) \ + R(sec_vlan_ts_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \ + R(sec_vlan_ts_cksum_ptype, \ + R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \ + R(sec_vlan_ts_cksum_ptype_rss, \ + R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \ + R(sec_vlan_ts_mark, R_SEC_F | RX_VLAN_F | TS_F | MARK_F) \ + R(sec_vlan_ts_mark_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F) \ + R(sec_vlan_ts_mark_ptype, \ + R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \ + R(sec_vlan_ts_mark_ptype_rss, \ + R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \ + R(sec_vlan_ts_mark_cksum, \ + R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \ + R(sec_vlan_ts_mark_cksum_rss, \ + R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \ + R(sec_vlan_ts_mark_cksum_ptype, \ + R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \ + R(sec_vlan_ts_mark_cksum_ptype_rss, \ + R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + +#define NIX_RX_FASTPATH_MODES \ + NIX_RX_FASTPATH_MODES_0_15 \ + NIX_RX_FASTPATH_MODES_16_31 \ + NIX_RX_FASTPATH_MODES_32_47 \ + NIX_RX_FASTPATH_MODES_48_63 \ + NIX_RX_FASTPATH_MODES_64_79 \ + NIX_RX_FASTPATH_MODES_80_95 \ + NIX_RX_FASTPATH_MODES_96_111 \ + NIX_RX_FASTPATH_MODES_112_127 + +#define R(name, flags) \ uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_##name( \ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ - \ uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_mseg_##name( \ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ - \ uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_##name( \ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ - \ uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_mseg_##name( \ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); NIX_RX_FASTPATH_MODES #undef R +#define NIX_RX_RECV(fn, flags) \ + uint16_t __rte_noinline __rte_hot fn( \ + void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \ + { \ + return cn9k_nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags)); \ + } + +#define NIX_RX_RECV_MSEG(fn, flags) NIX_RX_RECV(fn, flags | NIX_RX_MULTI_SEG_F) + +#define NIX_RX_RECV_VEC(fn, flags) \ + uint16_t __rte_noinline __rte_hot fn( \ + void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \ + { \ + return cn9k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts, \ + (flags)); \ + } + +#define NIX_RX_RECV_VEC_MSEG(fn, flags) \ + NIX_RX_RECV_VEC(fn, flags | NIX_RX_MULTI_SEG_F) + #endif /* __CN9K_RX_H__ */