X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fcnxk%2Fcn10k_rx.h;h=94c1f1e84fbfa99f910539ab65e52b15379dc60c;hb=1e1bfd078e581c155cd5a2b5c9191d87714dfc93;hp=236a1dca6e4d410af9e7d62ca450985117bb3d18;hpb=6c75d42006b7868754c28b9429685c9b8e0bb3cc;p=dpdk.git diff --git a/drivers/net/cnxk/cn10k_rx.h b/drivers/net/cnxk/cn10k_rx.h index 236a1dca6e..94c1f1e84f 100644 --- a/drivers/net/cnxk/cn10k_rx.h +++ b/drivers/net/cnxk/cn10k_rx.h @@ -23,6 +23,7 @@ * Defining it from backwards to denote its been * not used as offload flags to pick function */ +#define NIX_RX_REAS_F BIT(12) #define NIX_RX_VWQE_F BIT(13) #define NIX_RX_MULTI_SEG_F BIT(14) #define CPT_RX_WQE_F BIT(15) @@ -36,6 +37,38 @@ (((f) & NIX_RX_VWQE_F) ? \ (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) + (o)) : \ (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) + (o))) +#define CQE_PTR_DIFF(b, i, o, f) \ + (((f) & NIX_RX_VWQE_F) ? \ + (uint64_t *)(((uintptr_t)((uint64_t *)(b))[i]) - (o)) : \ + (uint64_t *)(((uintptr_t)(b)) + CQE_SZ(i) - (o))) + +#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +static inline void +nix_mbuf_validate_next(struct rte_mbuf *m) +{ + if (m->nb_segs == 1 && m->next) { + rte_panic("mbuf->next[%p] valid when mbuf->nb_segs is %d", + m->next, m->nb_segs); + } +} +#else +static inline void +nix_mbuf_validate_next(struct rte_mbuf *m) +{ + RTE_SET_USED(m); +} +#endif + +#define NIX_RX_SEC_REASSEMBLY_F \ + (NIX_RX_REAS_F | NIX_RX_OFFLOAD_SECURITY_F) + +static inline rte_eth_ip_reassembly_dynfield_t * +cnxk_ip_reassembly_dynfield(struct rte_mbuf *mbuf, + int ip_reassembly_dynfield_offset) +{ + return RTE_MBUF_DYNFIELD(mbuf, ip_reassembly_dynfield_offset, + rte_eth_ip_reassembly_dynfield_t *); +} union mbuf_initializer { struct { @@ -104,19 +137,348 @@ nix_sec_flush_meta(uintptr_t laddr, uint16_t lmt_id, uint8_t loff, roc_lmt_submit_steorl(lmt_id, pa); } +static struct rte_mbuf * +nix_sec_attach_frags(const struct cpt_parse_hdr_s *hdr, + struct cn10k_inb_priv_data *inb_priv, + const uint64_t mbuf_init) +{ + struct rte_mbuf *head, *mbuf, *mbuf_prev; + uint32_t offset = hdr->w2.fi_offset; + union nix_rx_parse_u *frag_rx; + struct cpt_frag_info_s *finfo; + uint64_t *frag_ptr = NULL; + uint64_t ol_flags; + uint16_t frag_size; + uint16_t rlen; + uint64_t *wqe; + int off; + + off = inb_priv->reass_dynfield_off; + ol_flags = BIT_ULL(inb_priv->reass_dynflag_bit); + ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD; + + /* offset of 0 implies 256B, otherwise it implies offset*8B */ + offset = (((offset - 1) & 0x1f) + 1) * 8; + finfo = RTE_PTR_ADD(hdr, offset + hdr->w2.fi_pad); + + /* Frag-0: */ + wqe = (uint64_t *)(rte_be_to_cpu_64(hdr->wqe_ptr)); + rlen = ((*(wqe + 10)) >> 16) & 0xFFFF; + + frag_rx = (union nix_rx_parse_u *)(wqe + 1); + frag_size = rlen + frag_rx->lcptr - frag_rx->laptr; + frag_rx->pkt_lenm1 = frag_size - 1; + + mbuf = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf)); + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init; + mbuf->data_len = frag_size; + mbuf->pkt_len = frag_size; + mbuf->ol_flags = ol_flags; + mbuf->next = NULL; + head = mbuf; + mbuf_prev = mbuf; + /* Update dynamic field with userdata */ + *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata; + + cnxk_ip_reassembly_dynfield(head, off)->nb_frags = hdr->w0.num_frags - 1; + cnxk_ip_reassembly_dynfield(head, off)->next_frag = NULL; + + /* Frag-1: */ + if (hdr->w0.num_frags > 1) { + wqe = (uint64_t *)(rte_be_to_cpu_64(hdr->frag1_wqe_ptr)); + rlen = ((*(wqe + 10)) >> 16) & 0xFFFF; + + frag_rx = (union nix_rx_parse_u *)(wqe + 1); + frag_size = rlen + frag_rx->lcptr - frag_rx->laptr; + frag_rx->pkt_lenm1 = frag_size - 1; + + mbuf = (struct rte_mbuf *)((uintptr_t)wqe - + sizeof(struct rte_mbuf)); + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init; + mbuf->data_len = frag_size; + mbuf->pkt_len = frag_size; + mbuf->ol_flags = ol_flags; + mbuf->next = NULL; + + /* Update dynamic field with userdata */ + *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata; + + cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags = + hdr->w0.num_frags - 2; + cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = NULL; + cnxk_ip_reassembly_dynfield(mbuf_prev, off)->next_frag = mbuf; + mbuf_prev = mbuf; + } + + /* Frag-2: */ + if (hdr->w0.num_frags > 2) { + frag_ptr = (uint64_t *)(finfo + 1); + wqe = (uint64_t *)(rte_be_to_cpu_64(*frag_ptr)); + rlen = ((*(wqe + 10)) >> 16) & 0xFFFF; + + frag_rx = (union nix_rx_parse_u *)(wqe + 1); + frag_size = rlen + frag_rx->lcptr - frag_rx->laptr; + frag_rx->pkt_lenm1 = frag_size - 1; + + mbuf = (struct rte_mbuf *)((uintptr_t)wqe - + sizeof(struct rte_mbuf)); + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init; + mbuf->data_len = frag_size; + mbuf->pkt_len = frag_size; + mbuf->ol_flags = ol_flags; + mbuf->next = NULL; + + /* Update dynamic field with userdata */ + *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata; + + cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags = + hdr->w0.num_frags - 3; + cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = NULL; + cnxk_ip_reassembly_dynfield(mbuf_prev, off)->next_frag = mbuf; + mbuf_prev = mbuf; + } + + /* Frag-3: */ + if (hdr->w0.num_frags > 3) { + wqe = (uint64_t *)(rte_be_to_cpu_64(*(frag_ptr + 1))); + rlen = ((*(wqe + 10)) >> 16) & 0xFFFF; + + frag_rx = (union nix_rx_parse_u *)(wqe + 1); + frag_size = rlen + frag_rx->lcptr - frag_rx->laptr; + frag_rx->pkt_lenm1 = frag_size - 1; + + mbuf = (struct rte_mbuf *)((uintptr_t)wqe - + sizeof(struct rte_mbuf)); + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init; + mbuf->data_len = frag_size; + mbuf->pkt_len = frag_size; + mbuf->ol_flags = ol_flags; + mbuf->next = NULL; + + /* Update dynamic field with userdata */ + *rte_security_dynfield(mbuf) = (uint64_t)inb_priv->userdata; + + cnxk_ip_reassembly_dynfield(mbuf, off)->nb_frags = + hdr->w0.num_frags - 4; + cnxk_ip_reassembly_dynfield(mbuf, off)->next_frag = NULL; + cnxk_ip_reassembly_dynfield(mbuf_prev, off)->next_frag = mbuf; + } + return head; +} + +static struct rte_mbuf * +nix_sec_reassemble_frags(const struct cpt_parse_hdr_s *hdr, uint64_t cq_w1, + uint64_t cq_w5, uint64_t mbuf_init) +{ + uint32_t fragx_sum, pkt_hdr_len, l3_hdr_size; + uint32_t offset = hdr->w2.fi_offset; + union nix_rx_parse_u *inner_rx; + uint16_t rlen, data_off, b_off; + union nix_rx_parse_u *frag_rx; + struct cpt_frag_info_s *finfo; + struct rte_mbuf *head, *mbuf; + uint64_t *frag_ptr = NULL; + rte_iova_t *inner_iova; + uint16_t frag_size; + uint64_t *wqe; + + /* Base data offset */ + b_off = mbuf_init & 0xFFFFUL; + mbuf_init &= ~0xFFFFUL; + + /* offset of 0 implies 256B, otherwise it implies offset*8B */ + offset = (((offset - 1) & 0x1f) + 1) * 8; + finfo = RTE_PTR_ADD(hdr, offset + hdr->w2.fi_pad); + + /* Frag-0: */ + wqe = (uint64_t *)rte_be_to_cpu_64(hdr->wqe_ptr); + inner_rx = (union nix_rx_parse_u *)(wqe + 1); + inner_iova = (rte_iova_t *)*(wqe + 9); + + /* Update only the upper 28-bits from meta pkt parse info */ + *((uint64_t *)inner_rx) = ((*((uint64_t *)inner_rx) & ((1ULL << 36) - 1)) | + (cq_w1 & ~((1ULL << 36) - 1))); + + rlen = ((*(wqe + 10)) >> 16) & 0xFFFF; + frag_size = rlen + ((cq_w5 >> 16) & 0xFF) - (cq_w5 & 0xFF); + fragx_sum = rte_be_to_cpu_16(finfo->w1.frag_size0); + pkt_hdr_len = frag_size - fragx_sum; + + mbuf = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf)); + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | b_off; + mbuf->data_len = frag_size; + head = mbuf; + + if (inner_rx->lctype == NPC_LT_LC_IP) { + struct rte_ipv4_hdr *hdr = (struct rte_ipv4_hdr *) + RTE_PTR_ADD(inner_iova, inner_rx->lcptr); + + l3_hdr_size = (hdr->version_ihl & 0xf) << 2; + } else { + struct rte_ipv6_hdr *hdr = (struct rte_ipv6_hdr *) + RTE_PTR_ADD(inner_iova, inner_rx->lcptr); + size_t ext_len = sizeof(struct rte_ipv6_hdr); + uint8_t *nxt_hdr = (uint8_t *)hdr; + int nh = hdr->proto; + + l3_hdr_size = 0; + while (nh != -EINVAL) { + nxt_hdr += ext_len; + l3_hdr_size += ext_len; + nh = rte_ipv6_get_next_ext(nxt_hdr, nh, &ext_len); + } + } + + /* Frag-1: */ + wqe = (uint64_t *)(rte_be_to_cpu_64(hdr->frag1_wqe_ptr)); + frag_size = rte_be_to_cpu_16(finfo->w1.frag_size1); + frag_rx = (union nix_rx_parse_u *)(wqe + 1); + + mbuf->next = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf)); + mbuf = mbuf->next; + data_off = b_off + frag_rx->lcptr + l3_hdr_size; + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off; + mbuf->data_len = frag_size; + fragx_sum += frag_size; + + /* Frag-2: */ + if (hdr->w0.num_frags > 2) { + frag_ptr = (uint64_t *)(finfo + 1); + wqe = (uint64_t *)(rte_be_to_cpu_64(*frag_ptr)); + frag_size = rte_be_to_cpu_16(finfo->w1.frag_size2); + frag_rx = (union nix_rx_parse_u *)(wqe + 1); + + mbuf->next = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf)); + mbuf = mbuf->next; + data_off = b_off + frag_rx->lcptr + l3_hdr_size; + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off; + mbuf->data_len = frag_size; + fragx_sum += frag_size; + } + + /* Frag-3: */ + if (hdr->w0.num_frags > 3) { + wqe = (uint64_t *)(rte_be_to_cpu_64(*(frag_ptr + 1))); + frag_size = rte_be_to_cpu_16(finfo->w1.frag_size3); + frag_rx = (union nix_rx_parse_u *)(wqe + 1); + + mbuf->next = (struct rte_mbuf *)((uintptr_t)wqe - sizeof(struct rte_mbuf)); + mbuf = mbuf->next; + data_off = b_off + frag_rx->lcptr + l3_hdr_size; + *(uint64_t *)(&mbuf->rearm_data) = mbuf_init | data_off; + mbuf->data_len = frag_size; + fragx_sum += frag_size; + } + + if (inner_rx->lctype == NPC_LT_LC_IP) { + struct rte_ipv4_hdr *hdr = (struct rte_ipv4_hdr *) + RTE_PTR_ADD(inner_iova, inner_rx->lcptr); + + hdr->fragment_offset = 0; + hdr->total_length = rte_cpu_to_be_16(fragx_sum + l3_hdr_size); + hdr->hdr_checksum = 0; + hdr->hdr_checksum = rte_ipv4_cksum(hdr); + + inner_rx->pkt_lenm1 = pkt_hdr_len + fragx_sum - 1; + } else { + /* Remove the frag header by moving header 8 bytes forward */ + struct rte_ipv6_hdr *hdr = (struct rte_ipv6_hdr *) + RTE_PTR_ADD(inner_iova, inner_rx->lcptr); + + hdr->payload_len = rte_cpu_to_be_16(fragx_sum + l3_hdr_size - + 8 - sizeof(struct rte_ipv6_hdr)); + + rte_memcpy(rte_pktmbuf_mtod_offset(head, void *, 8), + rte_pktmbuf_mtod(head, void *), + inner_rx->lcptr + sizeof(struct rte_ipv6_hdr)); + + inner_rx->pkt_lenm1 = pkt_hdr_len + fragx_sum - 8 - 1; + head->data_len -= 8; + head->data_off += 8; + } + mbuf->next = NULL; + head->pkt_len = inner_rx->pkt_lenm1 + 1; + head->nb_segs = hdr->w0.num_frags; + + return head; +} + static __rte_always_inline struct rte_mbuf * -nix_sec_meta_to_mbuf_sc(uint64_t cq_w1, const uint64_t sa_base, uintptr_t laddr, - uint8_t *loff, struct rte_mbuf *mbuf, uint16_t data_off) +nix_sec_meta_to_mbuf_sc(uint64_t cq_w1, uint64_t cq_w5, const uint64_t sa_base, + uintptr_t laddr, uint8_t *loff, struct rte_mbuf *mbuf, + uint16_t data_off, const uint16_t flags, + const uint64_t mbuf_init) { const void *__p = (void *)((uintptr_t)mbuf + (uint16_t)data_off); const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)__p; struct cn10k_inb_priv_data *inb_priv; - struct rte_mbuf *inner; + struct rte_mbuf *inner = NULL; + uint64_t res_w1; uint32_t sa_idx; + uint16_t uc_cc; + uint32_t len; void *inb_sa; uint64_t w0; - if (cq_w1 & BIT(11)) { + if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11))) { + /* Get SPI from CPT_PARSE_S's cookie(already swapped) */ + w0 = hdr->w0.u64; + sa_idx = w0 >> 32; + + inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx); + inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa); + + if (!hdr->w0.num_frags) { + /* No Reassembly or inbound error */ + inner = (struct rte_mbuf *) + (rte_be_to_cpu_64(hdr->wqe_ptr) - + sizeof(struct rte_mbuf)); + + /* Update dynamic field with userdata */ + *rte_security_dynfield(inner) = + (uint64_t)inb_priv->userdata; + + /* CPT result(struct cpt_cn10k_res_s) is at + * after first IOVA in meta + */ + res_w1 = *((uint64_t *)(&inner[1]) + 10); + uc_cc = res_w1 & 0xFF; + + /* Calculate inner packet length */ + len = ((res_w1 >> 16) & 0xFFFF) + hdr->w2.il3_off - + sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7); + inner->pkt_len = len; + inner->data_len = len; + *(uint64_t *)(&inner->rearm_data) = mbuf_init; + + inner->ol_flags = ((uc_cc == CPT_COMP_WARN) ? + RTE_MBUF_F_RX_SEC_OFFLOAD : + (RTE_MBUF_F_RX_SEC_OFFLOAD | + RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED)); + } else if (!(hdr->w0.err_sum) && !(hdr->w0.reas_sts)) { + /* Reassembly success */ + inner = nix_sec_reassemble_frags(hdr, cq_w1, cq_w5, + mbuf_init); + + /* Update dynamic field with userdata */ + *rte_security_dynfield(inner) = + (uint64_t)inb_priv->userdata; + + /* Assume success */ + inner->ol_flags = RTE_MBUF_F_RX_SEC_OFFLOAD; + } else { + /* Reassembly failure */ + inner = nix_sec_attach_frags(hdr, inb_priv, mbuf_init); + } + + /* Store meta in lmtline to free + * Assume all meta's from same aura. + */ + *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf; + *loff = *loff + 1; + + return inner; + } else if (cq_w1 & BIT(11)) { inner = (struct rte_mbuf *)(rte_be_to_cpu_64(hdr->wqe_ptr) - sizeof(struct rte_mbuf)); @@ -131,8 +493,24 @@ nix_sec_meta_to_mbuf_sc(uint64_t cq_w1, const uint64_t sa_base, uintptr_t laddr, *rte_security_dynfield(inner) = (uint64_t)inb_priv->userdata; /* Update l2 hdr length first */ - inner->pkt_len = (hdr->w2.il3_off - - sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7)); + + /* CPT result(struct cpt_cn10k_res_s) is at + * after first IOVA in meta + */ + res_w1 = *((uint64_t *)(&inner[1]) + 10); + uc_cc = res_w1 & 0xFF; + + /* Calculate inner packet length */ + len = ((res_w1 >> 16) & 0xFFFF) + hdr->w2.il3_off - + sizeof(struct cpt_parse_hdr_s) - (w0 & 0x7); + inner->pkt_len = len; + inner->data_len = len; + *(uint64_t *)(&inner->rearm_data) = mbuf_init; + + inner->ol_flags = ((uc_cc == CPT_COMP_WARN) ? + RTE_MBUF_F_RX_SEC_OFFLOAD : + (RTE_MBUF_F_RX_SEC_OFFLOAD | + RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED)); /* Store meta in lmtline to free * Assume all meta's from same aura. @@ -148,18 +526,22 @@ nix_sec_meta_to_mbuf_sc(uint64_t cq_w1, const uint64_t sa_base, uintptr_t laddr, return inner; } + return mbuf; } #if defined(RTE_ARCH_ARM64) static __rte_always_inline struct rte_mbuf * -nix_sec_meta_to_mbuf(uint64_t cq_w1, uintptr_t sa_base, uintptr_t laddr, - uint8_t *loff, struct rte_mbuf *mbuf, uint16_t data_off, - uint8x16_t *rx_desc_field1, uint64_t *ol_flags) +nix_sec_meta_to_mbuf(uint64_t cq_w1, uint64_t cq_w5, uintptr_t sa_base, + uintptr_t laddr, uint8_t *loff, struct rte_mbuf *mbuf, + uint16_t data_off, uint8x16_t *rx_desc_field1, + uint64_t *ol_flags, const uint16_t flags, + uint64x2_t *rearm) { const void *__p = (void *)((uintptr_t)mbuf + (uint16_t)data_off); const struct cpt_parse_hdr_s *hdr = (const struct cpt_parse_hdr_s *)__p; + uint64_t mbuf_init = vgetq_lane_u64(*rearm, 0); struct cn10k_inb_priv_data *inb_priv; struct rte_mbuf *inner; uint64_t *sg, res_w1; @@ -168,7 +550,100 @@ nix_sec_meta_to_mbuf(uint64_t cq_w1, uintptr_t sa_base, uintptr_t laddr, uint16_t len; uint64_t w0; - if (cq_w1 & BIT(11)) { + if ((flags & NIX_RX_REAS_F) && (cq_w1 & BIT(11))) { + w0 = hdr->w0.u64; + sa_idx = w0 >> 32; + + /* Get SPI from CPT_PARSE_S's cookie(already swapped) */ + w0 = hdr->w0.u64; + sa_idx = w0 >> 32; + + inb_sa = roc_nix_inl_ot_ipsec_inb_sa(sa_base, sa_idx); + inb_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd(inb_sa); + + /* Clear checksum flags */ + *ol_flags &= ~(RTE_MBUF_F_RX_L4_CKSUM_MASK | + RTE_MBUF_F_RX_IP_CKSUM_MASK); + + if (!hdr->w0.num_frags) { + /* No Reassembly or inbound error */ + inner = (struct rte_mbuf *) + (rte_be_to_cpu_64(hdr->wqe_ptr) - + sizeof(struct rte_mbuf)); + /* Update dynamic field with userdata */ + *rte_security_dynfield(inner) = + (uint64_t)inb_priv->userdata; + + /* CPT result(struct cpt_cn10k_res_s) is at + * after first IOVA in meta + */ + sg = (uint64_t *)(inner + 1); + res_w1 = sg[10]; + + /* Clear checksum flags and update security flag */ + *ol_flags &= ~(RTE_MBUF_F_RX_L4_CKSUM_MASK | + RTE_MBUF_F_RX_IP_CKSUM_MASK); + *ol_flags |= + (((res_w1 & 0xFF) == CPT_COMP_WARN) ? + RTE_MBUF_F_RX_SEC_OFFLOAD : + (RTE_MBUF_F_RX_SEC_OFFLOAD | + RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED)); + /* Calculate inner packet length */ + len = ((res_w1 >> 16) & 0xFFFF) + + hdr->w2.il3_off - + sizeof(struct cpt_parse_hdr_s) - + (w0 & 0x7); + /* Update pkt_len and data_len */ + *rx_desc_field1 = + vsetq_lane_u16(len, *rx_desc_field1, 2); + *rx_desc_field1 = + vsetq_lane_u16(len, *rx_desc_field1, 4); + + } else if (!(hdr->w0.err_sum) && !(hdr->w0.reas_sts)) { + /* Reassembly success */ + inner = nix_sec_reassemble_frags(hdr, cq_w1, cq_w5, + mbuf_init); + sg = (uint64_t *)(inner + 1); + res_w1 = sg[10]; + + /* Update dynamic field with userdata */ + *rte_security_dynfield(inner) = + (uint64_t)inb_priv->userdata; + + /* Assume success */ + *ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD; + + /* Update pkt_len and data_len */ + *rx_desc_field1 = vsetq_lane_u16(inner->pkt_len, + *rx_desc_field1, 2); + *rx_desc_field1 = vsetq_lane_u16(inner->data_len, + *rx_desc_field1, 4); + + /* Data offset might be updated */ + mbuf_init = *(uint64_t *)(&inner->rearm_data); + *rearm = vsetq_lane_u64(mbuf_init, *rearm, 0); + } else { + /* Reassembly failure */ + inner = nix_sec_attach_frags(hdr, inb_priv, mbuf_init); + *ol_flags |= inner->ol_flags; + + /* Update pkt_len and data_len */ + *rx_desc_field1 = vsetq_lane_u16(inner->pkt_len, + *rx_desc_field1, 2); + *rx_desc_field1 = vsetq_lane_u16(inner->data_len, + *rx_desc_field1, 4); + } + + /* Store meta in lmtline to free + * Assume all meta's from same aura. + */ + *(uint64_t *)(laddr + (*loff << 3)) = (uint64_t)mbuf; + *loff = *loff + 1; + + /* Return inner mbuf */ + return inner; + + } else if (cq_w1 & BIT(11)) { inner = (struct rte_mbuf *)(rte_be_to_cpu_64(hdr->wqe_ptr) - sizeof(struct rte_mbuf)); /* Get SPI from CPT_PARSE_S's cookie(already swapped) */ @@ -283,7 +758,7 @@ nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf, sg = *(const uint64_t *)(rx + 1); nb_segs = (sg >> 48) & 0x3; - if (nb_segs == 1) { + if (nb_segs == 1 && !(flags & NIX_RX_SEC_REASSEMBLY_F)) { mbuf->next = NULL; return; } @@ -346,30 +821,10 @@ cn10k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, ol_flags |= RTE_MBUF_F_RX_RSS_HASH; } - /* Process Security packets */ - if (flag & NIX_RX_OFFLOAD_SECURITY_F) { - if (w1 & BIT(11)) { - /* CPT result(struct cpt_cn10k_res_s) is at - * after first IOVA in meta - */ - const uint64_t *sg = (const uint64_t *)(mbuf + 1); - const uint64_t res_w1 = sg[10]; - const uint16_t uc_cc = res_w1 & 0xFF; - - /* Rlen */ - len = ((res_w1 >> 16) & 0xFFFF) + mbuf->pkt_len; - ol_flags |= ((uc_cc == CPT_COMP_WARN) ? - RTE_MBUF_F_RX_SEC_OFFLOAD : - (RTE_MBUF_F_RX_SEC_OFFLOAD | - RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED)); - } else { - if (flag & NIX_RX_OFFLOAD_CHECKSUM_F) - ol_flags |= nix_rx_olflags_get(lookup_mem, w1); - } - } else { - if (flag & NIX_RX_OFFLOAD_CHECKSUM_F) - ol_flags |= nix_rx_olflags_get(lookup_mem, w1); - } + /* Skip rx ol flags extraction for Security packets */ + if ((!(flag & NIX_RX_SEC_REASSEMBLY_F) || !(w1 & BIT(11))) && + flag & NIX_RX_OFFLOAD_CHECKSUM_F) + ol_flags |= nix_rx_olflags_get(lookup_mem, w1); if (flag & NIX_RX_OFFLOAD_VLAN_STRIP_F) { if (rx->vtag0_gone) { @@ -385,10 +840,15 @@ cn10k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, if (flag & NIX_RX_OFFLOAD_MARK_UPDATE_F) ol_flags = nix_update_match_id(rx->match_id, ol_flags, mbuf); - mbuf->ol_flags = ol_flags; - mbuf->pkt_len = len; - mbuf->data_len = len; - *(uint64_t *)(&mbuf->rearm_data) = val; + /* Packet data length and ol flags is already updated for sec */ + if (flag & NIX_RX_SEC_REASSEMBLY_F && w1 & BIT_ULL(11)) { + mbuf->ol_flags |= ol_flags; + } else { + mbuf->ol_flags = ol_flags; + mbuf->pkt_len = len; + mbuf->data_len = len; + *(uint64_t *)(&mbuf->rearm_data) = val; + } if (flag & NIX_RX_MULTI_SEG_F) /* @@ -398,8 +858,6 @@ cn10k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag, * Hence, flag argument is not required. */ nix_cqe_xtract_mseg(rx, mbuf, val, 0); - else - mbuf->next = NULL; } static inline uint16_t @@ -509,9 +967,11 @@ cn10k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts, /* Translate meta to mbuf */ if (flags & NIX_RX_OFFLOAD_SECURITY_F) { const uint64_t cq_w1 = *((const uint64_t *)cq + 1); + const uint64_t cq_w5 = *((const uint64_t *)cq + 5); - mbuf = nix_sec_meta_to_mbuf_sc(cq_w1, sa_base, laddr, - &loff, mbuf, data_off); + mbuf = nix_sec_meta_to_mbuf_sc(cq_w1, cq_w5, sa_base, laddr, + &loff, mbuf, data_off, + flags, mbuf_init); } cn10k_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init, @@ -547,10 +1007,11 @@ cn10k_nix_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts, plt_write64((wdata | nb_pkts), rxq->cq_door); /* Free remaining meta buffers if any */ - if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff) { + if (flags & NIX_RX_OFFLOAD_SECURITY_F && loff) nix_sec_flush_meta(laddr, lmt_id + lnum, loff, aura_handle); - plt_io_wmb(); - } + + if (flags & NIX_RX_OFFLOAD_SECURITY_F) + rte_io_wmb(); return nb_pkts; } @@ -674,17 +1135,66 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, cq0 = (uintptr_t)&mbufs[packets]; } - /* Prefetch N desc ahead */ - rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 4, 64, flags)); - rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 5, 64, flags)); - rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 6, 64, flags)); - rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 7, 64, flags)); - - /* Get NIX_RX_SG_S for size and buffer pointer */ - cq0_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 0, 64, flags)); - cq1_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 1, 64, flags)); - cq2_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 2, 64, flags)); - cq3_w8 = vld1q_u64(CQE_PTR_OFF(cq0, 3, 64, flags)); + if (flags & NIX_RX_VWQE_F) { + if (pkts - packets > 4) { + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, + 4, 0, flags)); + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, + 5, 0, flags)); + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, + 6, 0, flags)); + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, + 7, 0, flags)); + + if (likely(pkts - packets > 8)) { + rte_prefetch1(CQE_PTR_OFF(cq0, + 8, 0, flags)); + rte_prefetch1(CQE_PTR_OFF(cq0, + 9, 0, flags)); + rte_prefetch1(CQE_PTR_OFF(cq0, + 10, 0, flags)); + rte_prefetch1(CQE_PTR_OFF(cq0, + 11, 0, flags)); + if (pkts - packets > 12) { + rte_prefetch1(CQE_PTR_OFF(cq0, + 12, 0, flags)); + rte_prefetch1(CQE_PTR_OFF(cq0, + 13, 0, flags)); + rte_prefetch1(CQE_PTR_OFF(cq0, + 14, 0, flags)); + rte_prefetch1(CQE_PTR_OFF(cq0, + 15, 0, flags)); + } + } + + rte_prefetch0(CQE_PTR_DIFF(cq0, + 4, RTE_PKTMBUF_HEADROOM, flags)); + rte_prefetch0(CQE_PTR_DIFF(cq0, + 5, RTE_PKTMBUF_HEADROOM, flags)); + rte_prefetch0(CQE_PTR_DIFF(cq0, + 6, RTE_PKTMBUF_HEADROOM, flags)); + rte_prefetch0(CQE_PTR_DIFF(cq0, + 7, RTE_PKTMBUF_HEADROOM, flags)); + + if (likely(pkts - packets > 8)) { + rte_prefetch0(CQE_PTR_DIFF(cq0, + 8, RTE_PKTMBUF_HEADROOM, flags)); + rte_prefetch0(CQE_PTR_DIFF(cq0, + 9, RTE_PKTMBUF_HEADROOM, flags)); + rte_prefetch0(CQE_PTR_DIFF(cq0, + 10, RTE_PKTMBUF_HEADROOM, flags)); + rte_prefetch0(CQE_PTR_DIFF(cq0, + 11, RTE_PKTMBUF_HEADROOM, flags)); + } + } + } else { + if (pkts - packets > 4) { + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 4, 64, flags)); + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 5, 64, flags)); + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 6, 64, flags)); + rte_prefetch_non_temporal(CQE_PTR_OFF(cq0, 7, 64, flags)); + } + } if (!(flags & NIX_RX_VWQE_F)) { /* Get NIX_RX_SG_S for size and buffer pointer */ @@ -819,25 +1329,40 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, /* Translate meta to mbuf */ if (flags & NIX_RX_OFFLOAD_SECURITY_F) { + uint64_t cq0_w5 = *(uint64_t *)(cq0 + CQE_SZ(0) + 40); + uint64_t cq1_w5 = *(uint64_t *)(cq0 + CQE_SZ(1) + 40); + uint64_t cq2_w5 = *(uint64_t *)(cq0 + CQE_SZ(2) + 40); + uint64_t cq3_w5 = *(uint64_t *)(cq0 + CQE_SZ(3) + 40); + + /* Initialize rearm data when reassembly is enabled as + * data offset might change. + */ + if (flags & NIX_RX_REAS_F) { + rearm0 = vdupq_n_u64(mbuf_initializer); + rearm1 = vdupq_n_u64(mbuf_initializer); + rearm2 = vdupq_n_u64(mbuf_initializer); + rearm3 = vdupq_n_u64(mbuf_initializer); + } + /* Checksum ol_flags will be cleared if mbuf is meta */ - mbuf0 = nix_sec_meta_to_mbuf(cq0_w1, sa_base, laddr, + mbuf0 = nix_sec_meta_to_mbuf(cq0_w1, cq0_w5, sa_base, laddr, &loff, mbuf0, d_off, &f0, - &ol_flags0); + &ol_flags0, flags, &rearm0); mbuf01 = vsetq_lane_u64((uint64_t)mbuf0, mbuf01, 0); - mbuf1 = nix_sec_meta_to_mbuf(cq1_w1, sa_base, laddr, + mbuf1 = nix_sec_meta_to_mbuf(cq1_w1, cq1_w5, sa_base, laddr, &loff, mbuf1, d_off, &f1, - &ol_flags1); + &ol_flags1, flags, &rearm1); mbuf01 = vsetq_lane_u64((uint64_t)mbuf1, mbuf01, 1); - mbuf2 = nix_sec_meta_to_mbuf(cq2_w1, sa_base, laddr, + mbuf2 = nix_sec_meta_to_mbuf(cq2_w1, cq2_w5, sa_base, laddr, &loff, mbuf2, d_off, &f2, - &ol_flags2); + &ol_flags2, flags, &rearm2); mbuf23 = vsetq_lane_u64((uint64_t)mbuf2, mbuf23, 0); - mbuf3 = nix_sec_meta_to_mbuf(cq3_w1, sa_base, laddr, + mbuf3 = nix_sec_meta_to_mbuf(cq3_w1, cq3_w5, sa_base, laddr, &loff, mbuf3, d_off, &f3, - &ol_flags3); + &ol_flags3, flags, &rearm3); mbuf23 = vsetq_lane_u64((uint64_t)mbuf3, mbuf23, 1); } @@ -995,19 +1520,18 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, nix_cqe_xtract_mseg((union nix_rx_parse_u *) (CQE_PTR_OFF(cq0, 3, 8, flags)), mbuf3, mbuf_initializer, flags); - } else { - /* Update that no more segments */ - mbuf0->next = NULL; - mbuf1->next = NULL; - mbuf2->next = NULL; - mbuf3->next = NULL; } - /* Prefetch mbufs */ - roc_prefetch_store_keep(mbuf0); - roc_prefetch_store_keep(mbuf1); - roc_prefetch_store_keep(mbuf2); - roc_prefetch_store_keep(mbuf3); + /* Mark mempool obj as "get" as it is alloc'ed by NIX */ + RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1); + RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1); + + nix_mbuf_validate_next(mbuf0); + nix_mbuf_validate_next(mbuf1); + nix_mbuf_validate_next(mbuf2); + nix_mbuf_validate_next(mbuf3); packets += NIX_DESCS_PER_LOOP; @@ -1294,6 +1818,7 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, R(sec_vlan_ts_mark_cksum_ptype_rss, \ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) + #define NIX_RX_FASTPATH_MODES \ NIX_RX_FASTPATH_MODES_0_15 \ NIX_RX_FASTPATH_MODES_16_31 \ @@ -1302,7 +1827,7 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, NIX_RX_FASTPATH_MODES_64_79 \ NIX_RX_FASTPATH_MODES_80_95 \ NIX_RX_FASTPATH_MODES_96_111 \ - NIX_RX_FASTPATH_MODES_112_127 + NIX_RX_FASTPATH_MODES_112_127 \ #define R(name, flags) \ uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_##name( \ @@ -1312,6 +1837,14 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts, uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_##name( \ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_mseg_##name( \ + void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ + uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_##name( \ + void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ + uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_mseg_##name(\ + void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ + uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_vec_##name( \ + void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \ + uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_reas_vec_mseg_##name( \ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); NIX_RX_FASTPATH_MODES