- uint32_t flags;
-
- flags = flags2_0xf(rxcmp1);
- /* IP Checksum */
- if (likely(IS_IP_NONTUNNEL_PKT(flags))) {
- if (unlikely(RX_CMP_IP_CS_ERROR(rxcmp1)))
- mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
- else
- mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
- } else if (IS_IP_TUNNEL_PKT(flags)) {
- if (unlikely(RX_CMP_IP_OUTER_CS_ERROR(rxcmp1) ||
- RX_CMP_IP_CS_ERROR(rxcmp1)))
- mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
- else
- mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
- } else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) {
- mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
- }
-
- /* L4 Checksum */
- if (likely(IS_L4_NONTUNNEL_PKT(flags))) {
- if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1)))
- mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
- else
- mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
- } else if (IS_L4_TUNNEL_PKT(flags)) {
- if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1)))
- mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
- else
- mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
- if (unlikely(RX_CMP_L4_OUTER_CS_ERR2(rxcmp1))) {
- mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_BAD;
- } else if (unlikely(IS_L4_TUNNEL_PKT_ONLY_INNER_L4_CS
- (flags))) {
- mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_UNKNOWN;
- } else {
- mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_GOOD;
- }
- } else if (unlikely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) {
- mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
- }
+ const uint8x16_t shuf_msk = {
+ 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */
+ 2, 3, 0xFF, 0xFF, /* pkt_len */
+ 2, 3, /* data_len */
+ 0xFF, 0xFF, /* vlan_tci (zeroes) */
+ 12, 13, 14, 15 /* rss hash */
+ };
+ const uint32x4_t flags_type_mask = {
+ RX_PKT_CMPL_FLAGS_ITYPE_MASK,
+ RX_PKT_CMPL_FLAGS_ITYPE_MASK,
+ RX_PKT_CMPL_FLAGS_ITYPE_MASK,
+ RX_PKT_CMPL_FLAGS_ITYPE_MASK
+ };
+ const uint32x4_t flags2_mask1 = {
+ RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN |
+ RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC,
+ RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN |
+ RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC,
+ RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN |
+ RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC,
+ RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN |
+ RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC
+ };
+ const uint32x4_t flags2_mask2 = {
+ RX_PKT_CMPL_FLAGS2_IP_TYPE,
+ RX_PKT_CMPL_FLAGS2_IP_TYPE,
+ RX_PKT_CMPL_FLAGS2_IP_TYPE,
+ RX_PKT_CMPL_FLAGS2_IP_TYPE
+ };
+ const uint32x4_t rss_mask = {
+ RX_PKT_CMPL_FLAGS_RSS_VALID,
+ RX_PKT_CMPL_FLAGS_RSS_VALID,
+ RX_PKT_CMPL_FLAGS_RSS_VALID,
+ RX_PKT_CMPL_FLAGS_RSS_VALID
+ };
+ const uint32x4_t flags2_index_mask = {
+ 0x1F, 0x1F, 0x1F, 0x1F
+ };
+ const uint32x4_t flags2_error_mask = {
+ 0xF, 0xF, 0xF, 0xF
+ };
+ uint32x4_t flags_type, flags2, index, errors, rss_flags;
+ uint32x4_t tmp, ptype_idx;
+ uint64x2_t t0, t1;
+ uint32_t ol_flags;
+
+ /* Compute packet type table indexes for four packets */
+ t0 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp[0], mm_rxcmp[1]));
+ t1 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp[2], mm_rxcmp[3]));
+
+ flags_type = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(t0),
+ vget_low_u64(t1)));
+ ptype_idx =
+ vshrq_n_u32(vandq_u32(flags_type, flags_type_mask), 9);
+
+ t0 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp1[0], mm_rxcmp1[1]));
+ t1 = vreinterpretq_u64_u32(vzip1q_u32(mm_rxcmp1[2], mm_rxcmp1[3]));
+
+ flags2 = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(t0),
+ vget_low_u64(t1)));
+
+ ptype_idx = vorrq_u32(ptype_idx,
+ vshrq_n_u32(vandq_u32(flags2, flags2_mask1), 2));
+ ptype_idx = vorrq_u32(ptype_idx,
+ vshrq_n_u32(vandq_u32(flags2, flags2_mask2), 7));
+
+ /* Extract RSS valid flags for four packets. */
+ rss_flags = vshrq_n_u32(vandq_u32(flags_type, rss_mask), 9);
+
+ flags2 = vandq_u32(flags2, flags2_index_mask);
+
+ /* Extract errors_v2 fields for four packets. */
+ t0 = vreinterpretq_u64_u32(vzip2q_u32(mm_rxcmp1[0], mm_rxcmp1[1]));
+ t1 = vreinterpretq_u64_u32(vzip2q_u32(mm_rxcmp1[2], mm_rxcmp1[3]));
+
+ errors = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(t0),
+ vget_low_u64(t1)));
+
+ /* Compute ol_flags and checksum error indexes for four packets. */
+ errors = vandq_u32(vshrq_n_u32(errors, 4), flags2_error_mask);
+ errors = vandq_u32(errors, flags2);
+
+ index = vbicq_u32(flags2, errors);
+
+ /* Update mbuf rearm_data for four packets. */
+ GET_OL_FLAGS(rss_flags, index, errors, 0, ol_flags);
+ vst1q_u32((uint32_t *)&mbuf[0]->rearm_data,
+ vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2));
+ GET_OL_FLAGS(rss_flags, index, errors, 1, ol_flags);
+ vst1q_u32((uint32_t *)&mbuf[1]->rearm_data,
+ vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2));
+ GET_OL_FLAGS(rss_flags, index, errors, 2, ol_flags);
+ vst1q_u32((uint32_t *)&mbuf[2]->rearm_data,
+ vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2));
+ GET_OL_FLAGS(rss_flags, index, errors, 3, ol_flags);
+ vst1q_u32((uint32_t *)&mbuf[3]->rearm_data,
+ vsetq_lane_u32(ol_flags, vreinterpretq_u32_u64(mb_init), 2));
+
+ /* Update mbuf rx_descriptor_fields1 for four packets. */
+ GET_DESC_FIELDS(mm_rxcmp[0], mm_rxcmp1[0], shuf_msk, ptype_idx, 0, tmp);
+ vst1q_u32((uint32_t *)&mbuf[0]->rx_descriptor_fields1, tmp);
+ GET_DESC_FIELDS(mm_rxcmp[1], mm_rxcmp1[1], shuf_msk, ptype_idx, 1, tmp);
+ vst1q_u32((uint32_t *)&mbuf[1]->rx_descriptor_fields1, tmp);
+ GET_DESC_FIELDS(mm_rxcmp[2], mm_rxcmp1[2], shuf_msk, ptype_idx, 2, tmp);
+ vst1q_u32((uint32_t *)&mbuf[2]->rx_descriptor_fields1, tmp);
+ GET_DESC_FIELDS(mm_rxcmp[3], mm_rxcmp1[3], shuf_msk, ptype_idx, 3, tmp);
+ vst1q_u32((uint32_t *)&mbuf[3]->rx_descriptor_fields1, tmp);