X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fixgbe%2Fixgbe_rxtx_vec.c;h=50407043f869a9e16b5de164c1eb85c53cc5bb19;hb=c2b96070527380d9b1bb3d9994324f82b5b37cf8;hp=cf25a53fcf4d8190657125463f184c69ff7102dd;hpb=33dd07136f14f81df2760be57a61df521b1e4c84;p=dpdk.git diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec.c b/drivers/net/ixgbe/ixgbe_rxtx_vec.c index cf25a53fcf..50407043f8 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec.c @@ -140,19 +140,11 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) */ #ifdef RTE_IXGBE_RX_OLFLAGS_ENABLE -#ifndef RTE_NEXT_ABI -#define OLFLAGS_MASK ((uint16_t)(PKT_RX_VLAN_PKT | PKT_RX_IPV4_HDR |\ - PKT_RX_IPV4_HDR_EXT | PKT_RX_IPV6_HDR |\ - PKT_RX_IPV6_HDR_EXT)) -#define PTYPE_SHIFT (1) -#endif /* RTE_NEXT_ABI */ - #define VTAG_SHIFT (3) static inline void desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) { -#ifdef RTE_NEXT_ABI __m128i ptype0, ptype1, vtag0, vtag1; union { uint16_t e[4]; @@ -190,50 +182,6 @@ desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) vtag1 = _mm_or_si128(ptype0, vtag1); vol.dword = _mm_cvtsi128_si64(vtag1); -#else - __m128i ptype0, ptype1, vtag0, vtag1; - union { - uint16_t e[4]; - uint64_t dword; - } vol; - - /* pkt type + vlan olflags mask */ - const __m128i pkttype_msk = _mm_set_epi16( - 0x0000, 0x0000, 0x0000, 0x0000, - OLFLAGS_MASK, OLFLAGS_MASK, OLFLAGS_MASK, OLFLAGS_MASK); - - /* mask everything except rss type */ - const __m128i rsstype_msk = _mm_set_epi16( - 0x0000, 0x0000, 0x0000, 0x0000, - 0x000F, 0x000F, 0x000F, 0x000F); - - /* rss type to PKT_RX_RSS_HASH translation */ - const __m128i rss_flags = _mm_set_epi8(PKT_RX_FDIR, 0, 0, 0, - 0, 0, 0, PKT_RX_RSS_HASH, - PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, 0, - PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 0); - - ptype0 = _mm_unpacklo_epi16(descs[0], descs[1]); - ptype1 = _mm_unpacklo_epi16(descs[2], descs[3]); - vtag0 = _mm_unpackhi_epi16(descs[0], descs[1]); - vtag1 = _mm_unpackhi_epi16(descs[2], descs[3]); - - ptype1 = _mm_unpacklo_epi32(ptype0, ptype1); - vtag1 = _mm_unpacklo_epi32(vtag0, vtag1); - - ptype0 = _mm_and_si128(ptype1, rsstype_msk); - ptype0 = _mm_shuffle_epi8(rss_flags, ptype0); - - ptype1 = _mm_slli_epi16(ptype1, PTYPE_SHIFT); - vtag1 = _mm_srli_epi16(vtag1, VTAG_SHIFT); - - ptype1 = _mm_or_si128(ptype1, vtag1); - ptype1 = _mm_and_si128(ptype1, pkttype_msk); - - ptype0 = _mm_or_si128(ptype0, ptype1); - - vol.dword = _mm_cvtsi128_si64(ptype0); -#endif /* RTE_NEXT_ABI */ rx_pkts[0]->ol_flags = vol.e[0]; rx_pkts[1]->ol_flags = vol.e[1]; @@ -245,13 +193,13 @@ desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) #endif /* - * vPMD receive routine, now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST) - * in one loop + * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * * Notice: - * - nb_pkts < RTE_IXGBE_VPMD_RX_BURST, just return no packet - * - nb_pkts > RTE_IXGBE_VPMD_RX_BURST, only scan RTE_IXGBE_VPMD_RX_BURST + * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST * numbers of DD bit + * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two * - don't support ol_flags for rss and csum err */ static inline uint16_t @@ -264,7 +212,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, int pos; uint64_t var; __m128i shuf_msk; -#ifdef RTE_NEXT_ABI __m128i crc_adjust = _mm_set_epi16( 0, 0, 0, /* ignore non-length fields */ -rxq->crc_len, /* sub crc on data_len */ @@ -273,21 +220,12 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 0, 0 /* ignore pkt_type field */ ); __m128i dd_check, eop_check; - __m128i desc_mask = _mm_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFF07F0); -#else - __m128i crc_adjust = _mm_set_epi16( - 0, 0, 0, 0, /* ignore non-length fields */ - 0, /* ignore high-16bits of pkt_len */ - -rxq->crc_len, /* sub crc on pkt_len */ - -rxq->crc_len, /* sub crc on data_len */ - 0 /* ignore pkt_type field */ - ); - __m128i dd_check, eop_check; -#endif /* RTE_NEXT_ABI */ - if (unlikely(nb_pkts < RTE_IXGBE_VPMD_RX_BURST)) - return 0; + /* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */ + nb_pkts = RTE_MIN(nb_pkts, RTE_IXGBE_MAX_RX_BURST); + + /* nb_pkts has to be floor-aligned to RTE_IXGBE_DESCS_PER_LOOP */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_IXGBE_DESCS_PER_LOOP); /* Just the act of getting into the function from the application is * going to cost about 7 cycles */ @@ -313,34 +251,20 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL); /* mask to shuffle from desc. to mbuf */ -#ifdef RTE_NEXT_ABI shuf_msk = _mm_set_epi8( 7, 6, 5, 4, /* octet 4~7, 32bits rss */ 15, 14, /* octet 14~15, low 16 bits vlan_macip */ 13, 12, /* octet 12~13, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ 13, 12, /* octet 12~13, low 16 bits pkt_len */ - 0xFF, 0xFF, /* skip high 16 bits pkt_type */ - 1, /* octet 1, 8 bits pkt_type field */ - 0 /* octet 0, 4 bits offset 4 pkt_type field */ - ); -#else - shuf_msk = _mm_set_epi8( - 7, 6, 5, 4, /* octet 4~7, 32bits rss */ - 0xFF, 0xFF, /* skip high 16 bits vlan_macip, zero out */ - 15, 14, /* octet 14~15, low 16 bits vlan_macip */ - 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ - 13, 12, /* octet 12~13, low 16 bits pkt_len */ - 13, 12, /* octet 12~13, 16 bits data_len */ - 0xFF, 0xFF /* skip pkt_type field */ + 0xFF, 0xFF, /* skip 32 bit pkt_type */ + 0xFF, 0xFF ); -#endif /* RTE_NEXT_ABI */ /* Cache is empty -> need to scan the buffer rings, but first move * the next 'n' mbufs into the cache */ sw_ring = &rxq->sw_ring[rxq->rx_tail]; -#ifdef RTE_NEXT_ABI /* A. load 4 packet in one loop * [A*. mask out 4 unused dirty field in desc] * B. copy 4 mbuf point from swring to rx_pkts @@ -348,62 +272,17 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, * [C*. extract the end-of-packet bit, if requested] * D. fill info. from desc to mbuf */ -#else - /* A. load 4 packet in one loop - * B. copy 4 mbuf point from swring to rx_pkts - * C. calc the number of DD bits among the 4 packets - * [C*. extract the end-of-packet bit, if requested] - * D. fill info. from desc to mbuf - */ -#endif /* RTE_NEXT_ABI */ - for (pos = 0, nb_pkts_recd = 0; pos < RTE_IXGBE_VPMD_RX_BURST; + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; pos += RTE_IXGBE_DESCS_PER_LOOP, rxdp += RTE_IXGBE_DESCS_PER_LOOP) { -#ifdef RTE_NEXT_ABI - __m128i descs0[RTE_IXGBE_DESCS_PER_LOOP]; -#endif /* RTE_NEXT_ABI */ __m128i descs[RTE_IXGBE_DESCS_PER_LOOP]; __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; __m128i zero, staterr, sterr_tmp1, sterr_tmp2; __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */ - if (split_packet) { - rte_prefetch0(&rx_pkts[pos]->cacheline1); - rte_prefetch0(&rx_pkts[pos + 1]->cacheline1); - rte_prefetch0(&rx_pkts[pos + 2]->cacheline1); - rte_prefetch0(&rx_pkts[pos + 3]->cacheline1); - } - /* B.1 load 1 mbuf point */ mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]); -#ifdef RTE_NEXT_ABI - /* Read desc statuses backwards to avoid race condition */ - /* A.1 load 4 pkts desc */ - descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); - - /* B.2 copy 2 mbuf point into rx_pkts */ - _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); - - /* B.1 load 1 mbuf point */ - mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); - - descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); - /* B.1 load 2 mbuf point */ - descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); - descs0[0] = _mm_loadu_si128((__m128i *)(rxdp)); - - /* B.2 copy 2 mbuf point into rx_pkts */ - _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2); - - /* A* mask out 0~3 bits RSS type */ - descs[3] = _mm_and_si128(descs0[3], desc_mask); - descs[2] = _mm_and_si128(descs0[2], desc_mask); - - /* A* mask out 0~3 bits RSS type */ - descs[1] = _mm_and_si128(descs0[1], desc_mask); - descs[0] = _mm_and_si128(descs0[0], desc_mask); -#else /* Read desc statuses backwards to avoid race condition */ /* A.1 load 4 pkts desc */ descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); @@ -412,7 +291,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); /* B.1 load 1 mbuf point */ - mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos + 2]); + mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); /* B.1 load 2 mbuf point */ @@ -420,8 +299,14 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); /* B.2 copy 2 mbuf point into rx_pkts */ - _mm_storeu_si128((__m128i *)&rx_pkts[pos + 2], mbp2); -#endif /* RTE_NEXT_ABI */ + _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2); + + if (split_packet) { + rte_prefetch0(&rx_pkts[pos]->cacheline1); + rte_prefetch0(&rx_pkts[pos + 1]->cacheline1); + rte_prefetch0(&rx_pkts[pos + 2]->cacheline1); + rte_prefetch0(&rx_pkts[pos + 3]->cacheline1); + } /* avoid compiler reorder optimization */ rte_compiler_barrier(); @@ -430,27 +315,22 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk); pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk); + /* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk); + pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk); + /* C.1 4=>2 filter staterr info only */ sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]); /* C.1 4=>2 filter staterr info only */ sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); -#ifdef RTE_NEXT_ABI /* set ol_flags with vlan packet type */ - desc_to_olflags_v(descs0, &rx_pkts[pos]); -#else - /* set ol_flags with packet type and vlan tag */ desc_to_olflags_v(descs, &rx_pkts[pos]); -#endif /* RTE_NEXT_ABI */ /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust); - /* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk); - pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk); - /* C.2 get 4 pkts staterr value */ zero = _mm_xor_si128(dd_check, dd_check); staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2); @@ -518,13 +398,13 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, } /* - * vPMD receive routine, now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST) - * in one loop + * vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) * * Notice: - * - nb_pkts < RTE_IXGBE_VPMD_RX_BURST, just return no packet - * - nb_pkts > RTE_IXGBE_VPMD_RX_BURST, only scan RTE_IXGBE_VPMD_RX_BURST + * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST * numbers of DD bit + * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two * - don't support ol_flags for rss and csum err */ uint16_t @@ -538,12 +418,11 @@ static inline uint16_t reassemble_packets(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_bufs, uint16_t nb_bufs, uint8_t *split_flags) { - struct rte_mbuf *pkts[RTE_IXGBE_VPMD_RX_BURST]; /*finished pkts*/ + struct rte_mbuf *pkts[nb_bufs]; /*finished pkts*/ struct rte_mbuf *start = rxq->pkt_first_seg; struct rte_mbuf *end = rxq->pkt_last_seg; unsigned pkt_idx, buf_idx; - for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { if (end != NULL) { /* processing a split packet */ @@ -603,14 +482,17 @@ reassemble_packets(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_bufs, * * Notice: * - don't support ol_flags for rss and csum err - * - now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST) + * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST + * numbers of DD bit + * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two */ uint16_t ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct ixgbe_rx_queue *rxq = rx_queue; - uint8_t split_flags[RTE_IXGBE_VPMD_RX_BURST] = {0}; + uint8_t split_flags[RTE_IXGBE_MAX_RX_BURST] = {0}; /* get some new buffers */ uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts, @@ -735,8 +617,8 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint64_t rs = IXGBE_ADVTXD_DCMD_RS|DCMD_DTYP_FLAGS; int i; - if (unlikely(nb_pkts > RTE_IXGBE_VPMD_TX_BURST)) - nb_pkts = RTE_IXGBE_VPMD_TX_BURST; + /* cross rx_thresh boundary is not allowed */ + nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); if (txq->nb_tx_free < txq->tx_free_thresh) ixgbe_tx_free_bufs(txq);