X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fixgbe%2Fixgbe_rxtx_vec_sse.c;h=7610fd93db7436e06f30b71603a1b28bb212f903;hb=c5a49265fc232ac382d4609b264b57b3d65dbbe3;hp=586a261808c768ee1a65f5f2f441f8233edf312c;hpb=a8d0d473a0a89b3c50813e3e144e9a5377429f24;p=dpdk.git diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c index 586a261808..7610fd93db 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c @@ -3,7 +3,7 @@ */ #include -#include +#include #include #include "ixgbe_ethdev.h" @@ -132,9 +132,9 @@ desc_to_olflags_v_ipsec(__m128i descs[4], struct rte_mbuf **rx_pkts) static inline void desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, - struct rte_mbuf **rx_pkts) + uint16_t udp_p_flag, struct rte_mbuf **rx_pkts) { - __m128i ptype0, ptype1, vtag0, vtag1, csum; + __m128i ptype0, ptype1, vtag0, vtag1, csum, udp_csum_skip; __m128i rearm0, rearm1, rearm2, rearm3; /* mask everything except rss type */ @@ -161,6 +161,7 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16, IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP); + /* map vlan present (0x8), IPE (0x2), L4E (0x1) to ol_flags */ const __m128i vlan_csum_map_lo = _mm_set_epi8( 0, 0, 0, 0, @@ -182,12 +183,23 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, 0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t)); + /* mask everything except UDP header present if specified */ + const __m128i udp_hdr_p_msk = _mm_set_epi16 + (0, 0, 0, 0, + udp_p_flag, udp_p_flag, udp_p_flag, udp_p_flag); + + const __m128i udp_csum_bad_shuf = _mm_set_epi8 + (0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, ~(uint8_t)PKT_RX_L4_CKSUM_BAD, 0xFF); + ptype0 = _mm_unpacklo_epi16(descs[0], descs[1]); ptype1 = _mm_unpacklo_epi16(descs[2], descs[3]); vtag0 = _mm_unpackhi_epi16(descs[0], descs[1]); vtag1 = _mm_unpackhi_epi16(descs[2], descs[3]); ptype0 = _mm_unpacklo_epi32(ptype0, ptype1); + /* save the UDP header present information */ + udp_csum_skip = _mm_and_si128(ptype0, udp_hdr_p_msk); ptype0 = _mm_and_si128(ptype0, rsstype_msk); ptype0 = _mm_shuffle_epi8(rss_flags, ptype0); @@ -215,6 +227,15 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, vtag1 = _mm_or_si128(ptype0, vtag1); + /* convert the UDP header present 0x200 to 0x1 for aligning with each + * PKT_RX_L4_CKSUM_BAD value in low byte of 16 bits word ol_flag in + * vtag1 (4x16). Then mask out the bad checksum value by shuffle and + * bit-mask. + */ + udp_csum_skip = _mm_srli_epi16(udp_csum_skip, 9); + udp_csum_skip = _mm_shuffle_epi8(udp_csum_bad_shuf, udp_csum_skip); + vtag1 = _mm_and_si128(vtag1, udp_csum_skip); + /* * At this point, we have the 4 sets of flags in the low 64-bits * of vtag1 (4x16). @@ -302,13 +323,11 @@ desc_to_ptype_v(__m128i descs[4], uint16_t pkt_type_mask, get_packet_type(3, pkt_info, etqf_check, tunnel_check); } -/* +/** * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * * Notice: * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST - * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two */ static inline uint16_t @@ -343,9 +362,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, __m128i dd_check, eop_check; __m128i mbuf_init; uint8_t vlan_flags; - - /* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */ - nb_pkts = RTE_MIN(nb_pkts, RTE_IXGBE_MAX_RX_BURST); + uint16_t udp_p_flag = 0; /* Rx Descriptor UDP header present */ /* nb_pkts has to be floor-aligned to RTE_IXGBE_DESCS_PER_LOOP */ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_IXGBE_DESCS_PER_LOOP); @@ -370,6 +387,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) return 0; + if (rxq->rx_udp_csum_zero_err) + udp_p_flag = IXGBE_RXDADV_PKTTYPE_UDP; + /* 4 packets DD mask */ dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL); @@ -482,7 +502,8 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); /* set ol_flags with vlan packet type */ - desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]); + desc_to_olflags_v(descs, mbuf_init, vlan_flags, udp_p_flag, + &rx_pkts[pos]); #ifdef RTE_LIB_SECURITY if (unlikely(use_ipsec)) @@ -556,13 +577,11 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, return nb_pkts_recd; } -/* +/** * vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * * Notice: * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST - * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two */ uint16_t @@ -572,18 +591,16 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); } -/* +/** * vPMD receive routine that reassembles scattered packets * * Notice: * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST - * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two */ -uint16_t -ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) +static uint16_t +ixgbe_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) { struct ixgbe_rx_queue *rxq = rx_queue; uint8_t split_flags[RTE_IXGBE_MAX_RX_BURST] = {0}; @@ -615,6 +632,32 @@ ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, &split_flags[i]); } +/** + * vPMD receive routine that reassembles scattered packets. + */ +uint16_t +ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + uint16_t retval = 0; + + while (nb_pkts > RTE_IXGBE_MAX_RX_BURST) { + uint16_t burst; + + burst = ixgbe_recv_scattered_burst_vec(rx_queue, + rx_pkts + retval, + RTE_IXGBE_MAX_RX_BURST); + retval += burst; + nb_pkts -= burst; + if (burst < RTE_IXGBE_MAX_RX_BURST) + return retval; + } + + return retval + ixgbe_recv_scattered_burst_vec(rx_queue, + rx_pkts + retval, + nb_pkts); +} + static inline void vtx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf *pkt, uint64_t flags)