+ uint64_t cq0_w5 = *CQE_PTR_OFF(cq0, 0, 40, flags);
+ uint64_t cq1_w5 = *CQE_PTR_OFF(cq0, 1, 40, flags);
+ uint64_t cq2_w5 = *CQE_PTR_OFF(cq0, 2, 40, flags);
+ uint64_t cq3_w5 = *CQE_PTR_OFF(cq0, 3, 40, flags);
+ uintptr_t cpth0 = (uintptr_t)mbuf0 + d_off;
+ uintptr_t cpth1 = (uintptr_t)mbuf1 + d_off;
+ uintptr_t cpth2 = (uintptr_t)mbuf2 + d_off;
+ uintptr_t cpth3 = (uintptr_t)mbuf3 + d_off;
+
+ uint64x2_t inner0, inner1, inner2, inner3;
+ uint64x2_t wqe01, wqe23, sa01, sa23;
+ uint16x4_t lens, l2lens, ltypes;
+ uint8x8_t ucc;
+
+ inner0 = vld1q_u64((const uint64_t *)cpth0);
+ inner1 = vld1q_u64((const uint64_t *)cpth1);
+ inner2 = vld1q_u64((const uint64_t *)cpth2);
+ inner3 = vld1q_u64((const uint64_t *)cpth3);
+
+ /* Extract and reverse wqe pointers */
+ wqe01 = vzip2q_u64(inner0, inner1);
+ wqe23 = vzip2q_u64(inner2, inner3);
+ wqe01 = vrev64q_u8(wqe01);
+ wqe23 = vrev64q_u8(wqe23);
+ /* Adjust wqe pointers to point to mbuf */
+ wqe01 = vsubq_u64(wqe01,
+ vdupq_n_u64(sizeof(struct rte_mbuf)));
+ wqe23 = vsubq_u64(wqe23,
+ vdupq_n_u64(sizeof(struct rte_mbuf)));
+
+ /* Extract sa idx from cookie area and add to sa_base */
+ sa01 = vzip1q_u64(inner0, inner1);
+ sa23 = vzip1q_u64(inner2, inner3);
+
+ sa01 = vshrq_n_u64(sa01, 32);
+ sa23 = vshrq_n_u64(sa23, 32);
+ sa01 = vshlq_n_u64(sa01,
+ ROC_NIX_INL_OT_IPSEC_INB_SA_SZ_LOG2);
+ sa23 = vshlq_n_u64(sa23,
+ ROC_NIX_INL_OT_IPSEC_INB_SA_SZ_LOG2);
+ sa01 = vaddq_u64(sa01, vdupq_n_u64(sa_base));
+ sa23 = vaddq_u64(sa23, vdupq_n_u64(sa_base));
+
+ const uint8x16_t tbl = {
+ /* ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST */
+ 0,
+ /* ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM */
+ RTE_MBUF_F_RX_IP_CKSUM_BAD >> 1,
+ /* ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_AGAIN */
+ 0,
+ /* ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM */
+ (RTE_MBUF_F_RX_IP_CKSUM_GOOD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1,
+ /* ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM */
+ (RTE_MBUF_F_RX_IP_CKSUM_GOOD |
+ RTE_MBUF_F_RX_L4_CKSUM_BAD) >> 1,
+ /* ROC_IE_OT_UCC_SUCCESS_PKT_UDPESP_NZCSUM */
+ (RTE_MBUF_F_RX_IP_CKSUM_GOOD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1,
+ /* ROC_IE_OT_UCC_SUCCESS_PKT_UDP_ZEROCSUM */
+ (RTE_MBUF_F_RX_IP_CKSUM_GOOD |
+ RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1,
+ /* ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM */
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD >> 1,
+ /* HW_CCODE -> RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED */
+ 1, 0, 1, 1, 1, 1, 0, 1,
+ };
+
+ const int8x8_t err_off = {
+ /* UCC of significance starts from 0xF0 */
+ 0xF0,
+ /* Move HW_CCODE from 0:6 -> 8:14 */
+ -8,
+ 0xF0,
+ -8,
+ 0xF0,
+ -8,
+ 0xF0,
+ -8,
+ };
+
+ ucc = vdup_n_u8(0);
+ ucc = vset_lane_u16(*(uint16_t *)(cpth0 + 30), ucc, 0);
+ ucc = vset_lane_u16(*(uint16_t *)(cpth1 + 30), ucc, 1);
+ ucc = vset_lane_u16(*(uint16_t *)(cpth2 + 30), ucc, 2);
+ ucc = vset_lane_u16(*(uint16_t *)(cpth3 + 30), ucc, 3);
+ ucc = vsub_s8(ucc, err_off);
+ ucc = vqtbl1_u8(tbl, ucc);
+
+ RTE_BUILD_BUG_ON(NPC_LT_LC_IP != 2);
+ RTE_BUILD_BUG_ON(NPC_LT_LC_IP_OPT != 3);
+ RTE_BUILD_BUG_ON(NPC_LT_LC_IP6 != 4);
+ RTE_BUILD_BUG_ON(NPC_LT_LC_IP6_EXT != 5);
+
+ ltypes = vdup_n_u16(0);
+ ltypes = vset_lane_u16((cq0_w1 >> 40) & 0x6, ltypes, 0);
+ ltypes = vset_lane_u16((cq1_w1 >> 40) & 0x6, ltypes, 1);
+ ltypes = vset_lane_u16((cq2_w1 >> 40) & 0x6, ltypes, 2);
+ ltypes = vset_lane_u16((cq3_w1 >> 40) & 0x6, ltypes, 3);
+
+ /* Extract and reverse l3 length from IPv4/IPv6 hdr
+ * that is in same cacheline most probably as cpth.
+ */
+ cpth0 += ((cq0_w5 >> 16) & 0xFF) +
+ vget_lane_u16(ltypes, 0);
+ cpth1 += ((cq1_w5 >> 16) & 0xFF) +
+ vget_lane_u16(ltypes, 1);
+ cpth2 += ((cq2_w5 >> 16) & 0xFF) +
+ vget_lane_u16(ltypes, 2);
+ cpth3 += ((cq3_w5 >> 16) & 0xFF) +
+ vget_lane_u16(ltypes, 3);
+ lens = vdup_n_u16(0);
+ lens = vset_lane_u16(*(uint16_t *)cpth0, lens, 0);
+ lens = vset_lane_u16(*(uint16_t *)cpth1, lens, 1);
+ lens = vset_lane_u16(*(uint16_t *)cpth2, lens, 2);
+ lens = vset_lane_u16(*(uint16_t *)cpth3, lens, 3);
+ lens = vrev16_u8(lens);
+
+ /* Add l2 length to l3 lengths */
+ l2lens = vdup_n_u16(0);
+ l2lens = vset_lane_u16(((cq0_w5 >> 16) & 0xFF) -
+ (cq0_w5 & 0xFF),
+ l2lens, 0);
+ l2lens = vset_lane_u16(((cq1_w5 >> 16) & 0xFF) -
+ (cq1_w5 & 0xFF),
+ l2lens, 1);
+ l2lens = vset_lane_u16(((cq2_w5 >> 16) & 0xFF) -
+ (cq2_w5 & 0xFF),
+ l2lens, 2);
+ l2lens = vset_lane_u16(((cq3_w5 >> 16) & 0xFF) -
+ (cq3_w5 & 0xFF),
+ l2lens, 3);
+ lens = vadd_u16(lens, l2lens);
+
+ /* L3 header adjust */
+ const int8x8_t l3adj = {
+ 0, 0, 0, 0, 40, 0, 0, 0,
+ };
+ lens = vadd_u16(lens, vtbl1_u8(l3adj, ltypes));