X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fixgbe%2Fixgbe_rxtx_vec_sse.c;h=b65220f1bb1cb49cecb9c05df8abf0a6e7c54b31;hb=00fabc423db0ee5ed47da6e4c7d3c0c1e1a738b4;hp=d209bb224b0c248fbb651a5f5d4894b05c59b091;hpb=5ea4d4688dcea7add275ae2e3654878a1834465d;p=dpdk.git diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c index d209bb224b..b65220f1bb 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c @@ -122,6 +122,59 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); } +static inline void +desc_to_olflags_v_ipsec(__m128i descs[4], struct rte_mbuf **rx_pkts) +{ + __m128i sterr0, sterr1, sterr2, sterr3; + __m128i tmp1, tmp2, tmp3, tmp4; + __m128i rearm0, rearm1, rearm2, rearm3; + + const __m128i ipsec_sterr_msk = _mm_set_epi32( + 0, IXGBE_RXDADV_IPSEC_STATUS_SECP | + IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED, + 0, 0); + const __m128i ipsec_proc_msk = _mm_set_epi32( + 0, IXGBE_RXDADV_IPSEC_STATUS_SECP, 0, 0); + const __m128i ipsec_err_flag = _mm_set_epi32( + 0, PKT_RX_SEC_OFFLOAD_FAILED | PKT_RX_SEC_OFFLOAD, + 0, 0); + const __m128i ipsec_proc_flag = _mm_set_epi32( + 0, PKT_RX_SEC_OFFLOAD, 0, 0); + + rearm0 = _mm_load_si128((__m128i *)&rx_pkts[0]->rearm_data); + rearm1 = _mm_load_si128((__m128i *)&rx_pkts[1]->rearm_data); + rearm2 = _mm_load_si128((__m128i *)&rx_pkts[2]->rearm_data); + rearm3 = _mm_load_si128((__m128i *)&rx_pkts[3]->rearm_data); + sterr0 = _mm_and_si128(descs[0], ipsec_sterr_msk); + sterr1 = _mm_and_si128(descs[1], ipsec_sterr_msk); + sterr2 = _mm_and_si128(descs[2], ipsec_sterr_msk); + sterr3 = _mm_and_si128(descs[3], ipsec_sterr_msk); + tmp1 = _mm_cmpeq_epi32(sterr0, ipsec_sterr_msk); + tmp2 = _mm_cmpeq_epi32(sterr0, ipsec_proc_msk); + tmp3 = _mm_cmpeq_epi32(sterr1, ipsec_sterr_msk); + tmp4 = _mm_cmpeq_epi32(sterr1, ipsec_proc_msk); + sterr0 = _mm_or_si128(_mm_and_si128(tmp1, ipsec_err_flag), + _mm_and_si128(tmp2, ipsec_proc_flag)); + sterr1 = _mm_or_si128(_mm_and_si128(tmp3, ipsec_err_flag), + _mm_and_si128(tmp4, ipsec_proc_flag)); + tmp1 = _mm_cmpeq_epi32(sterr2, ipsec_sterr_msk); + tmp2 = _mm_cmpeq_epi32(sterr2, ipsec_proc_msk); + tmp3 = _mm_cmpeq_epi32(sterr3, ipsec_sterr_msk); + tmp4 = _mm_cmpeq_epi32(sterr3, ipsec_proc_msk); + sterr2 = _mm_or_si128(_mm_and_si128(tmp1, ipsec_err_flag), + _mm_and_si128(tmp2, ipsec_proc_flag)); + sterr3 = _mm_or_si128(_mm_and_si128(tmp3, ipsec_err_flag), + _mm_and_si128(tmp4, ipsec_proc_flag)); + rearm0 = _mm_or_si128(rearm0, sterr0); + rearm1 = _mm_or_si128(rearm1, sterr1); + rearm2 = _mm_or_si128(rearm2, sterr2); + rearm3 = _mm_or_si128(rearm3, sterr3); + _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); +} + static inline void desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, struct rte_mbuf **rx_pkts) @@ -232,6 +285,68 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); } +static inline uint32_t get_packet_type(int index, + uint32_t pkt_info, + uint32_t etqf_check, + uint32_t tunnel_check) +{ + if (etqf_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) + return RTE_PTYPE_UNKNOWN; + + if (tunnel_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) { + pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL; + return ptype_table_tn[pkt_info]; + } + + pkt_info &= IXGBE_PACKET_TYPE_MASK_82599; + return ptype_table[pkt_info]; +} + +static inline void +desc_to_ptype_v(__m128i descs[4], uint16_t pkt_type_mask, + struct rte_mbuf **rx_pkts) +{ + __m128i etqf_mask = _mm_set_epi64x(0x800000008000LL, 0x800000008000LL); + __m128i ptype_mask = _mm_set_epi32( + pkt_type_mask, pkt_type_mask, pkt_type_mask, pkt_type_mask); + __m128i tunnel_mask = + _mm_set_epi64x(0x100000001000LL, 0x100000001000LL); + + uint32_t etqf_check, tunnel_check, pkt_info; + + __m128i ptype0 = _mm_unpacklo_epi32(descs[0], descs[2]); + __m128i ptype1 = _mm_unpacklo_epi32(descs[1], descs[3]); + + /* interleave low 32 bits, + * now we have 4 ptypes in a XMM register + */ + ptype0 = _mm_unpacklo_epi32(ptype0, ptype1); + + /* create a etqf bitmask based on the etqf bit. */ + etqf_check = _mm_movemask_epi8(_mm_and_si128(ptype0, etqf_mask)); + + /* shift left by IXGBE_PACKET_TYPE_SHIFT, and apply ptype mask */ + ptype0 = _mm_and_si128(_mm_srli_epi32(ptype0, IXGBE_PACKET_TYPE_SHIFT), + ptype_mask); + + /* create a tunnel bitmask based on the tunnel bit */ + tunnel_check = _mm_movemask_epi8( + _mm_slli_epi32(_mm_and_si128(ptype0, tunnel_mask), 0x3)); + + pkt_info = _mm_extract_epi32(ptype0, 0); + rx_pkts[0]->packet_type = + get_packet_type(0, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 1); + rx_pkts[1]->packet_type = + get_packet_type(1, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 2); + rx_pkts[2]->packet_type = + get_packet_type(2, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 3); + rx_pkts[3]->packet_type = + get_packet_type(3, pkt_info, etqf_check, tunnel_check); +} + /* * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * @@ -248,6 +363,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, volatile union ixgbe_adv_rx_desc *rxdp; struct ixgbe_rx_entry *sw_ring; uint16_t nb_pkts_recd; + uint8_t use_ipsec = rxq->using_ipsec; int pos; uint64_t var; __m128i shuf_msk; @@ -411,6 +527,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, /* set ol_flags with vlan packet type */ desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]); + if (unlikely(use_ipsec)) + desc_to_olflags_v_ipsec(descs, rx_pkts); + /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust); @@ -461,6 +580,8 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, pkt_mb1); + desc_to_ptype_v(descs, rxq->pkt_type_mask, &rx_pkts[pos]); + /* C.4 calc avaialbe number of desc */ var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); nb_pkts_recd += var;