net/ixgbe: fix IPsec Rx with SSE

And small performance improvement

Fixes: 9a0752f498d2 ("net/ixgbe: enable inline IPsec")

Signed-off-by: Radu Nicolau <radu.nicolau@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
index a3b5b6b..486239b 100644 (file)
--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
@@ -126,54 +126,36 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
 static inline void
 desc_to_olflags_v_ipsec(__m128i descs[4], struct rte_mbuf **rx_pkts)
 {
-       __m128i sterr0, sterr1, sterr2, sterr3;
-       __m128i tmp1, tmp2, tmp3, tmp4;
-       __m128i rearm0, rearm1, rearm2, rearm3;
-
-       const __m128i ipsec_sterr_msk = _mm_set_epi32(
-               0, IXGBE_RXDADV_IPSEC_STATUS_SECP |
-                       IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED,
-               0, 0);
-       const __m128i ipsec_proc_msk  = _mm_set_epi32(
-               0, IXGBE_RXDADV_IPSEC_STATUS_SECP, 0, 0);
-       const __m128i ipsec_err_flag  = _mm_set_epi32(
-               0, PKT_RX_SEC_OFFLOAD_FAILED | PKT_RX_SEC_OFFLOAD,
-               0, 0);
-       const __m128i ipsec_proc_flag = _mm_set_epi32(
-               0, PKT_RX_SEC_OFFLOAD, 0, 0);
-
-       rearm0 = _mm_load_si128((__m128i *)&rx_pkts[0]->rearm_data);
-       rearm1 = _mm_load_si128((__m128i *)&rx_pkts[1]->rearm_data);
-       rearm2 = _mm_load_si128((__m128i *)&rx_pkts[2]->rearm_data);
-       rearm3 = _mm_load_si128((__m128i *)&rx_pkts[3]->rearm_data);
-       sterr0 = _mm_and_si128(descs[0], ipsec_sterr_msk);
-       sterr1 = _mm_and_si128(descs[1], ipsec_sterr_msk);
-       sterr2 = _mm_and_si128(descs[2], ipsec_sterr_msk);
-       sterr3 = _mm_and_si128(descs[3], ipsec_sterr_msk);
-       tmp1 = _mm_cmpeq_epi32(sterr0, ipsec_sterr_msk);
-       tmp2 = _mm_cmpeq_epi32(sterr0, ipsec_proc_msk);
-       tmp3 = _mm_cmpeq_epi32(sterr1, ipsec_sterr_msk);
-       tmp4 = _mm_cmpeq_epi32(sterr1, ipsec_proc_msk);
-       sterr0 = _mm_or_si128(_mm_and_si128(tmp1, ipsec_err_flag),
-                               _mm_and_si128(tmp2, ipsec_proc_flag));
-       sterr1 = _mm_or_si128(_mm_and_si128(tmp3, ipsec_err_flag),
-                               _mm_and_si128(tmp4, ipsec_proc_flag));
-       tmp1 = _mm_cmpeq_epi32(sterr2, ipsec_sterr_msk);
-       tmp2 = _mm_cmpeq_epi32(sterr2, ipsec_proc_msk);
-       tmp3 = _mm_cmpeq_epi32(sterr3, ipsec_sterr_msk);
-       tmp4 = _mm_cmpeq_epi32(sterr3, ipsec_proc_msk);
-       sterr2 = _mm_or_si128(_mm_and_si128(tmp1, ipsec_err_flag),
-                               _mm_and_si128(tmp2, ipsec_proc_flag));
-       sterr3 = _mm_or_si128(_mm_and_si128(tmp3, ipsec_err_flag),
-                               _mm_and_si128(tmp4, ipsec_proc_flag));
-       rearm0 = _mm_or_si128(rearm0, sterr0);
-       rearm1 = _mm_or_si128(rearm1, sterr1);
-       rearm2 = _mm_or_si128(rearm2, sterr2);
-       rearm3 = _mm_or_si128(rearm3, sterr3);
-       _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0);
-       _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1);
-       _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2);
-       _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
+       __m128i sterr, rearm, tmp_e, tmp_p;
+       uint32_t *rearm0 = (uint32_t *)rx_pkts[0]->rearm_data + 2;
+       uint32_t *rearm1 = (uint32_t *)rx_pkts[1]->rearm_data + 2;
+       uint32_t *rearm2 = (uint32_t *)rx_pkts[2]->rearm_data + 2;
+       uint32_t *rearm3 = (uint32_t *)rx_pkts[3]->rearm_data + 2;
+       const __m128i ipsec_sterr_msk =
+                       _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP |
+                                      IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED);
+       const __m128i ipsec_proc_msk  =
+                       _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP);
+       const __m128i ipsec_err_flag  =
+                       _mm_set1_epi32(PKT_RX_SEC_OFFLOAD_FAILED |
+                                      PKT_RX_SEC_OFFLOAD);
+       const __m128i ipsec_proc_flag = _mm_set1_epi32(PKT_RX_SEC_OFFLOAD);
+
+       rearm = _mm_set_epi32(*rearm3, *rearm2, *rearm1, *rearm0);
+       sterr = _mm_set_epi32(_mm_extract_epi32(descs[3], 2),
+                             _mm_extract_epi32(descs[2], 2),
+                             _mm_extract_epi32(descs[1], 2),
+                             _mm_extract_epi32(descs[0], 2));
+       sterr = _mm_and_si128(sterr, ipsec_sterr_msk);
+       tmp_e = _mm_cmpeq_epi32(sterr, ipsec_sterr_msk);
+       tmp_p = _mm_cmpeq_epi32(sterr, ipsec_proc_msk);
+       sterr = _mm_or_si128(_mm_and_si128(tmp_e, ipsec_err_flag),
+                               _mm_and_si128(tmp_p, ipsec_proc_flag));
+       rearm = _mm_or_si128(rearm, sterr);
+       *rearm0 = _mm_extract_epi32(rearm, 0);
+       *rearm1 = _mm_extract_epi32(rearm, 1);
+       *rearm2 = _mm_extract_epi32(rearm, 2);
+       *rearm3 = _mm_extract_epi32(rearm, 3);
 }
 #endif
 
@@ -533,7 +515,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 
 #ifdef RTE_LIBRTE_SECURITY
                if (unlikely(use_ipsec))
-                       desc_to_olflags_v_ipsec(descs, rx_pkts);
+                       desc_to_olflags_v_ipsec(descs, &rx_pkts[pos]);
 #endif
 
                /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */