X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fixgbe%2Fixgbe_rxtx_vec_sse.c;h=bb34b27168de8ae8edec2628aca9f34e2dea97d7;hb=75315881c23f3aa7c04fb19c50915e64dd97bd38;hp=203f092dfae834f4c2d2df07a7073be53caf98f3;hpb=631a2f530b6c1ea17e1c504f6170e6127439509a;p=dpdk.git diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c index 203f092dfa..bb34b27168 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c @@ -1,38 +1,9 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2015 Intel Corporation */ #include -#include +#include #include #include "ixgbe_ethdev.h" @@ -86,8 +57,8 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) mb0 = rxep[0].mbuf; mb1 = rxep[1].mbuf; - /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ - RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) != + /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) != offsetof(struct rte_mbuf, buf_addr) + 8); vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr)); vaddr1 = _mm_loadu_si128((__m128i *)&(mb1->buf_addr)); @@ -119,14 +90,51 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); /* Update the tail pointer on the NIC */ - IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); + IXGBE_PCI_REG_WC_WRITE(rxq->rdt_reg_addr, rx_id); } +#ifdef RTE_LIB_SECURITY +static inline void +desc_to_olflags_v_ipsec(__m128i descs[4], struct rte_mbuf **rx_pkts) +{ + __m128i sterr, rearm, tmp_e, tmp_p; + uint32_t *rearm0 = (uint32_t *)rx_pkts[0]->rearm_data + 2; + uint32_t *rearm1 = (uint32_t *)rx_pkts[1]->rearm_data + 2; + uint32_t *rearm2 = (uint32_t *)rx_pkts[2]->rearm_data + 2; + uint32_t *rearm3 = (uint32_t *)rx_pkts[3]->rearm_data + 2; + const __m128i ipsec_sterr_msk = + _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP | + IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED); + const __m128i ipsec_proc_msk = + _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP); + const __m128i ipsec_err_flag = + _mm_set1_epi32(RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED | + RTE_MBUF_F_RX_SEC_OFFLOAD); + const __m128i ipsec_proc_flag = _mm_set1_epi32(RTE_MBUF_F_RX_SEC_OFFLOAD); + + rearm = _mm_set_epi32(*rearm3, *rearm2, *rearm1, *rearm0); + sterr = _mm_set_epi32(_mm_extract_epi32(descs[3], 2), + _mm_extract_epi32(descs[2], 2), + _mm_extract_epi32(descs[1], 2), + _mm_extract_epi32(descs[0], 2)); + sterr = _mm_and_si128(sterr, ipsec_sterr_msk); + tmp_e = _mm_cmpeq_epi32(sterr, ipsec_sterr_msk); + tmp_p = _mm_cmpeq_epi32(sterr, ipsec_proc_msk); + sterr = _mm_or_si128(_mm_and_si128(tmp_e, ipsec_err_flag), + _mm_and_si128(tmp_p, ipsec_proc_flag)); + rearm = _mm_or_si128(rearm, sterr); + *rearm0 = _mm_extract_epi32(rearm, 0); + *rearm1 = _mm_extract_epi32(rearm, 1); + *rearm2 = _mm_extract_epi32(rearm, 2); + *rearm3 = _mm_extract_epi32(rearm, 3); +} +#endif + static inline void desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, - struct rte_mbuf **rx_pkts) + uint16_t udp_p_flag, struct rte_mbuf **rx_pkts) { - __m128i ptype0, ptype1, vtag0, vtag1, csum; + __m128i ptype0, ptype1, vtag0, vtag1, csum, udp_csum_skip; __m128i rearm0, rearm1, rearm2, rearm3; /* mask everything except rss type */ @@ -140,10 +148,10 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, 0x00FF, 0x00FF, 0x00FF, 0x00FF); /* map rss type to rss hash flag */ - const __m128i rss_flags = _mm_set_epi8(PKT_RX_FDIR, 0, 0, 0, - 0, 0, 0, PKT_RX_RSS_HASH, - PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, 0, - PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 0); + const __m128i rss_flags = _mm_set_epi8(RTE_MBUF_F_RX_FDIR, 0, 0, 0, + 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, + RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 0, + RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 0); /* mask everything except vlan present and l4/ip csum error */ const __m128i vlan_csum_msk = _mm_set_epi16( @@ -153,26 +161,36 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16, IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP); + /* map vlan present (0x8), IPE (0x2), L4E (0x1) to ol_flags */ const __m128i vlan_csum_map_lo = _mm_set_epi8( 0, 0, 0, 0, - vlan_flags | PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD, - vlan_flags | PKT_RX_IP_CKSUM_BAD, - vlan_flags | PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, - vlan_flags | PKT_RX_IP_CKSUM_GOOD, + vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD, + vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_BAD, + vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD, + vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_GOOD, 0, 0, 0, 0, - PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD, - PKT_RX_IP_CKSUM_BAD, - PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, - PKT_RX_IP_CKSUM_GOOD); + RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD, + RTE_MBUF_F_RX_IP_CKSUM_BAD, + RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD, + RTE_MBUF_F_RX_IP_CKSUM_GOOD); const __m128i vlan_csum_map_hi = _mm_set_epi8( 0, 0, 0, 0, - 0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, - PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), + 0, RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, + RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, 0, 0, 0, - 0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, - PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t)); + 0, RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, + RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t)); + + /* mask everything except UDP header present if specified */ + const __m128i udp_hdr_p_msk = _mm_set_epi16 + (0, 0, 0, 0, + udp_p_flag, udp_p_flag, udp_p_flag, udp_p_flag); + + const __m128i udp_csum_bad_shuf = _mm_set_epi8 + (0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, ~(uint8_t)RTE_MBUF_F_RX_L4_CKSUM_BAD, 0xFF); ptype0 = _mm_unpacklo_epi16(descs[0], descs[1]); ptype1 = _mm_unpacklo_epi16(descs[2], descs[3]); @@ -180,6 +198,8 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, vtag1 = _mm_unpackhi_epi16(descs[2], descs[3]); ptype0 = _mm_unpacklo_epi32(ptype0, ptype1); + /* save the UDP header present information */ + udp_csum_skip = _mm_and_si128(ptype0, udp_hdr_p_msk); ptype0 = _mm_and_si128(ptype0, rsstype_msk); ptype0 = _mm_shuffle_epi8(rss_flags, ptype0); @@ -207,6 +227,15 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, vtag1 = _mm_or_si128(ptype0, vtag1); + /* convert the UDP header present 0x200 to 0x1 for aligning with each + * RTE_MBUF_F_RX_L4_CKSUM_BAD value in low byte of 16 bits word ol_flag in + * vtag1 (4x16). Then mask out the bad checksum value by shuffle and + * bit-mask. + */ + udp_csum_skip = _mm_srli_epi16(udp_csum_skip, 9); + udp_csum_skip = _mm_shuffle_epi8(udp_csum_bad_shuf, udp_csum_skip); + vtag1 = _mm_and_si128(vtag1, udp_csum_skip); + /* * At this point, we have the 4 sets of flags in the low 64-bits * of vtag1 (4x16). @@ -216,26 +245,11 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, * appropriate flags means that we have to do a shift and blend for * each mbuf before we do the write. */ -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 - rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 8), 0x10); rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 6), 0x10); rearm2 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 4), 0x10); rearm3 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 2), 0x10); -#else - rearm0 = _mm_slli_si128(vtag1, 14); - rearm1 = _mm_slli_si128(vtag1, 12); - rearm2 = _mm_slli_si128(vtag1, 10); - rearm3 = _mm_slli_si128(vtag1, 8); - - rearm0 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm0, 48)); - rearm1 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm1, 48)); - rearm2 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm2, 48)); - rearm3 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm3, 48)); - -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ - /* write the rearm data and the olflags in one write */ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != offsetof(struct rte_mbuf, rearm_data) + 8); @@ -247,13 +261,73 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); } -/* +static inline uint32_t get_packet_type(int index, + uint32_t pkt_info, + uint32_t etqf_check, + uint32_t tunnel_check) +{ + if (etqf_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) + return RTE_PTYPE_UNKNOWN; + + if (tunnel_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) { + pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL; + return ptype_table_tn[pkt_info]; + } + + pkt_info &= IXGBE_PACKET_TYPE_MASK_82599; + return ptype_table[pkt_info]; +} + +static inline void +desc_to_ptype_v(__m128i descs[4], uint16_t pkt_type_mask, + struct rte_mbuf **rx_pkts) +{ + __m128i etqf_mask = _mm_set_epi64x(0x800000008000LL, 0x800000008000LL); + __m128i ptype_mask = _mm_set_epi32( + pkt_type_mask, pkt_type_mask, pkt_type_mask, pkt_type_mask); + __m128i tunnel_mask = + _mm_set_epi64x(0x100000001000LL, 0x100000001000LL); + + uint32_t etqf_check, tunnel_check, pkt_info; + + __m128i ptype0 = _mm_unpacklo_epi32(descs[0], descs[2]); + __m128i ptype1 = _mm_unpacklo_epi32(descs[1], descs[3]); + + /* interleave low 32 bits, + * now we have 4 ptypes in a XMM register + */ + ptype0 = _mm_unpacklo_epi32(ptype0, ptype1); + + /* create a etqf bitmask based on the etqf bit. */ + etqf_check = _mm_movemask_epi8(_mm_and_si128(ptype0, etqf_mask)); + + /* shift left by IXGBE_PACKET_TYPE_SHIFT, and apply ptype mask */ + ptype0 = _mm_and_si128(_mm_srli_epi32(ptype0, IXGBE_PACKET_TYPE_SHIFT), + ptype_mask); + + /* create a tunnel bitmask based on the tunnel bit */ + tunnel_check = _mm_movemask_epi8( + _mm_slli_epi32(_mm_and_si128(ptype0, tunnel_mask), 0x3)); + + pkt_info = _mm_extract_epi32(ptype0, 0); + rx_pkts[0]->packet_type = + get_packet_type(0, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 1); + rx_pkts[1]->packet_type = + get_packet_type(1, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 2); + rx_pkts[2]->packet_type = + get_packet_type(2, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 3); + rx_pkts[3]->packet_type = + get_packet_type(3, pkt_info, etqf_check, tunnel_check); +} + +/** * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * * Notice: * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST - * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two */ static inline uint16_t @@ -263,6 +337,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, volatile union ixgbe_adv_rx_desc *rxdp; struct ixgbe_rx_entry *sw_ring; uint16_t nb_pkts_recd; +#ifdef RTE_LIB_SECURITY + uint8_t use_ipsec = rxq->using_ipsec; +#endif int pos; uint64_t var; __m128i shuf_msk; @@ -285,9 +362,18 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, __m128i dd_check, eop_check; __m128i mbuf_init; uint8_t vlan_flags; + uint16_t udp_p_flag = 0; /* Rx Descriptor UDP header present */ - /* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */ - nb_pkts = RTE_MIN(nb_pkts, RTE_IXGBE_MAX_RX_BURST); + /* + * Under the circumstance that `rx_tail` wrap back to zero + * and the advance speed of `rx_tail` is greater than `rxrearm_start`, + * `rx_tail` will catch up with `rxrearm_start` and surpass it. + * This may cause some mbufs be reused by application. + * + * So we need to make some restrictions to ensure that + * `rx_tail` will not exceed `rxrearm_start`. + */ + nb_pkts = RTE_MIN(nb_pkts, RTE_IXGBE_RXQ_REARM_THRESH); /* nb_pkts has to be floor-aligned to RTE_IXGBE_DESCS_PER_LOOP */ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_IXGBE_DESCS_PER_LOOP); @@ -312,6 +398,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) return 0; + if (rxq->rx_udp_csum_zero_err) + udp_p_flag = IXGBE_RXDADV_PKTTYPE_UDP; + /* 4 packets DD mask */ dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL); @@ -350,7 +439,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, sw_ring = &rxq->sw_ring[rxq->rx_tail]; /* ensure these 2 flags are in the lower 8 bits */ - RTE_BUILD_BUG_ON((PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED) > UINT8_MAX); + RTE_BUILD_BUG_ON((RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED) > UINT8_MAX); vlan_flags = rxq->vlan_flags & UINT8_MAX; /* A. load 4 packet in one loop @@ -376,7 +465,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]); /* Read desc statuses backwards to avoid race condition */ - /* A.1 load 4 pkts desc */ + /* A.1 load desc[3] */ descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); rte_compiler_barrier(); @@ -388,9 +477,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); #endif + /* A.1 load desc[2-0] */ descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); rte_compiler_barrier(); - /* B.1 load 2 mbuf point */ descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); rte_compiler_barrier(); descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); @@ -424,7 +513,13 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); /* set ol_flags with vlan packet type */ - desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]); + desc_to_olflags_v(descs, mbuf_init, vlan_flags, udp_p_flag, + &rx_pkts[pos]); + +#ifdef RTE_LIB_SECURITY + if (unlikely(use_ipsec)) + desc_to_olflags_v_ipsec(descs, &rx_pkts[pos]); +#endif /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); @@ -456,7 +551,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, /* and with mask to extract bits, flipping 1-0 */ __m128i eop_bits = _mm_andnot_si128(staterr, eop_check); /* the staterr values are not in order, as the count - * count of dd bits doesn't care. However, for end of + * of dd bits doesn't care. However, for end of * packet tracking, we do care, so shuffle. This also * compresses the 32-bit values to 8-bit */ @@ -476,7 +571,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, pkt_mb1); - /* C.4 calc avaialbe number of desc */ + desc_to_ptype_v(descs, rxq->pkt_type_mask, &rx_pkts[pos]); + + /* C.4 calc available number of desc */ var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); nb_pkts_recd += var; if (likely(var != RTE_IXGBE_DESCS_PER_LOOP)) @@ -491,13 +588,11 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, return nb_pkts_recd; } -/* +/** * vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * * Notice: * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST - * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two */ uint16_t @@ -507,18 +602,16 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); } -/* +/** * vPMD receive routine that reassembles scattered packets * * Notice: * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST - * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two */ -uint16_t -ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) +static uint16_t +ixgbe_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) { struct ixgbe_rx_queue *rxq = rx_queue; uint8_t split_flags[RTE_IXGBE_MAX_RX_BURST] = {0}; @@ -544,18 +637,45 @@ ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, i++; if (i == nb_bufs) return nb_bufs; + rxq->pkt_first_seg = rx_pkts[i]; } return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, &split_flags[i]); } +/** + * vPMD receive routine that reassembles scattered packets. + */ +uint16_t +ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + uint16_t retval = 0; + + while (nb_pkts > RTE_IXGBE_MAX_RX_BURST) { + uint16_t burst; + + burst = ixgbe_recv_scattered_burst_vec(rx_queue, + rx_pkts + retval, + RTE_IXGBE_MAX_RX_BURST); + retval += burst; + nb_pkts -= burst; + if (burst < RTE_IXGBE_MAX_RX_BURST) + return retval; + } + + return retval + ixgbe_recv_scattered_burst_vec(rx_queue, + rx_pkts + retval, + nb_pkts); +} + static inline void vtx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf *pkt, uint64_t flags) { __m128i descriptor = _mm_set_epi64x((uint64_t)pkt->pkt_len << 46 | flags | pkt->data_len, - pkt->buf_physaddr + pkt->data_off); + pkt->buf_iova + pkt->data_off); _mm_store_si128((__m128i *)&txdp->read, descriptor); } @@ -631,30 +751,30 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; - IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, txq->tx_tail); + IXGBE_PCI_REG_WC_WRITE(txq->tdt_reg_addr, txq->tx_tail); return nb_pkts; } -static void __attribute__((cold)) +static void __rte_cold ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq) { _ixgbe_tx_queue_release_mbufs_vec(txq); } -void __attribute__((cold)) +void __rte_cold ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) { _ixgbe_rx_queue_release_mbufs_vec(rxq); } -static void __attribute__((cold)) +static void __rte_cold ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq) { _ixgbe_tx_free_swring_vec(txq); } -static void __attribute__((cold)) +static void __rte_cold ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) { _ixgbe_reset_tx_queue_vec(txq); @@ -666,19 +786,19 @@ static const struct ixgbe_txq_ops vec_txq_ops = { .reset = ixgbe_reset_tx_queue, }; -int __attribute__((cold)) +int __rte_cold ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) { return ixgbe_rxq_vec_setup_default(rxq); } -int __attribute__((cold)) +int __rte_cold ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq) { return ixgbe_txq_vec_setup_default(txq, &vec_txq_ops); } -int __attribute__((cold)) +int __rte_cold ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) { return ixgbe_rx_vec_dev_conf_condition_check_default(dev);