X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Ffm10k%2Ffm10k_rxtx_vec.c;h=27f3e43fffd2a8ded74747284fc2870d06b1d136;hb=c100fd464bb7;hp=06beca928bf7d16872778af686e363421033ee70;hpb=fb9066e479a6e21d851c66c20ec530c0b180247b;p=dpdk.git diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c index 06beca928b..27f3e43fff 100644 --- a/drivers/net/fm10k/fm10k_rxtx_vec.c +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c @@ -44,8 +44,6 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif -static void -fm10k_tx_queue_release_mbufs_vec(struct fm10k_tx_queue *txq); static void fm10k_reset_tx_queue(struct fm10k_tx_queue *txq); @@ -63,11 +61,19 @@ fm10k_reset_tx_queue(struct fm10k_tx_queue *txq); #define L3TYPE_SHIFT (4) /* L4 type shift */ #define L4TYPE_SHIFT (7) +/* HBO flag shift */ +#define HBOFLAG_SHIFT (10) +/* RXE flag shift */ +#define RXEFLAG_SHIFT (13) +/* IPE/L4E flag shift */ +#define L3L4EFLAG_SHIFT (14) +/* shift PKT_RX_L4_CKSUM_GOOD into one byte by 1 bit */ +#define CKSUM_SHIFT (1) static inline void fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) { - __m128i ptype0, ptype1, vtag0, vtag1; + __m128i ptype0, ptype1, vtag0, vtag1, eflag0, eflag1, cksumflag; union { uint16_t e[4]; uint64_t dword; @@ -83,12 +89,36 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) 0x0000, 0x0000, 0x0000, 0x0000, 0x000F, 0x000F, 0x000F, 0x000F); + /* mask for HBO and RXE flag flags */ + const __m128i rxe_msk = _mm_set_epi16( + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0001, 0x0001, 0x0001, 0x0001); + + /* mask the lower byte of ol_flags */ + const __m128i ol_flags_msk = _mm_set_epi16( + 0x0000, 0x0000, 0x0000, 0x0000, + 0x00FF, 0x00FF, 0x00FF, 0x00FF); + + const __m128i l3l4cksum_flag = _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD) >> CKSUM_SHIFT, + (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD) >> CKSUM_SHIFT, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> CKSUM_SHIFT, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> CKSUM_SHIFT); + + const __m128i rxe_flag = _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0); + /* map rss type to rss hash flag */ const __m128i rss_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 0); + /* Calculate RSS_hash and Vlan fields */ ptype0 = _mm_unpacklo_epi16(descs[0], descs[1]); ptype1 = _mm_unpacklo_epi16(descs[2], descs[3]); vtag0 = _mm_unpackhi_epi16(descs[0], descs[1]); @@ -99,10 +129,31 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) ptype0 = _mm_shuffle_epi8(rss_flags, ptype0); vtag1 = _mm_unpacklo_epi32(vtag0, vtag1); + eflag0 = vtag1; + cksumflag = vtag1; vtag1 = _mm_srli_epi16(vtag1, VP_SHIFT); vtag1 = _mm_and_si128(vtag1, pkttype_msk); vtag1 = _mm_or_si128(ptype0, vtag1); + + /* Process err flags, simply set RECIP_ERR bit if HBO/IXE is set */ + eflag1 = _mm_srli_epi16(eflag0, RXEFLAG_SHIFT); + eflag0 = _mm_srli_epi16(eflag0, HBOFLAG_SHIFT); + eflag0 = _mm_or_si128(eflag0, eflag1); + eflag0 = _mm_and_si128(eflag0, rxe_msk); + eflag0 = _mm_shuffle_epi8(rxe_flag, eflag0); + + vtag1 = _mm_or_si128(eflag0, vtag1); + + /* Process L4/L3 checksum error flags */ + cksumflag = _mm_srli_epi16(cksumflag, L3L4EFLAG_SHIFT); + cksumflag = _mm_shuffle_epi8(l3l4cksum_flag, cksumflag); + + /* clean the higher byte and shift back the flag bits */ + cksumflag = _mm_and_si128(cksumflag, ol_flags_msk); + cksumflag = _mm_slli_epi16(cksumflag, CKSUM_SHIFT); + vtag1 = _mm_or_si128(cksumflag, vtag1); + vol.dword = _mm_cvtsi128_si64(vtag1); rx_pkts[0]->ol_flags = vol.e[0]; @@ -111,6 +162,9 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) rx_pkts[3]->ol_flags = vol.e[3]; } +/* @note: When this function is changed, make corresponding change to + * fm10k_dev_supported_ptypes_get(). + */ static inline void fm10k_desc_to_pktype_v(__m128i descs[4], struct rte_mbuf **rx_pkts) { @@ -193,11 +247,8 @@ fm10k_rx_vec_condition_check(struct rte_eth_dev *dev) if (fconf->mode != RTE_FDIR_MODE_NONE) return -1; - /* - no csum error report support - * - no header split support - */ - if (rxmode->hw_ip_checksum == 1 || - rxmode->header_split == 1) + /* no header split support */ + if (rxmode->header_split == 1) return -1; return 0; @@ -365,7 +416,7 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, */ rxdp = rxq->hw_ring + next_dd; - _mm_prefetch((const void *)rxdp, _MM_HINT_T0); + rte_prefetch0(rxdp); /* See if we need to rearm the RX queue - gives the prefetch a bit * of time to act @@ -427,6 +478,7 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, /* Read desc statuses backwards to avoid race condition */ /* A.1 load 4 pkts desc */ descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); + rte_compiler_barrier(); /* B.2 copy 2 mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); @@ -435,8 +487,10 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]); descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); + rte_compiler_barrier(); /* B.1 load 2 mbuf point */ descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); + rte_compiler_barrier(); descs0[0] = _mm_loadu_si128((__m128i *)(rxdp)); /* B.2 copy 2 mbuf point into rx_pkts */ @@ -446,10 +500,10 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, rte_compiler_barrier(); if (split_packet) { - rte_prefetch0(&rx_pkts[pos]->cacheline1); - rte_prefetch0(&rx_pkts[pos + 1]->cacheline1); - rte_prefetch0(&rx_pkts[pos + 2]->cacheline1); - rte_prefetch0(&rx_pkts[pos + 3]->cacheline1); + rte_mbuf_prefetch_part2(rx_pkts[pos]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); } /* D.1 pkt 3,4 convert format from desc to pktmbuf */ @@ -565,8 +619,11 @@ fm10k_reassemble_packets(struct fm10k_rx_queue *rxq, if (!split_flags[buf_idx]) { /* it's the last packet of the set */ +#ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE start->hash = end->hash; start->ol_flags = end->ol_flags; + start->packet_type = end->packet_type; +#endif pkts[pkt_idx++] = start; start = end = NULL; } @@ -634,7 +691,6 @@ fm10k_recv_scattered_pkts_vec(void *rx_queue, } static const struct fm10k_txq_ops vec_txq_ops = { - .release_mbufs = fm10k_tx_queue_release_mbufs_vec, .reset = fm10k_reset_tx_queue, }; @@ -644,6 +700,19 @@ fm10k_txq_vec_setup(struct fm10k_tx_queue *txq) txq->ops = &vec_txq_ops; } +int __attribute__((cold)) +fm10k_tx_vec_condition_check(struct fm10k_tx_queue *txq) +{ + /* Vector TX can't offload any features yet */ + if ((txq->txq_flags & FM10K_SIMPLE_TX_FLAG) != FM10K_SIMPLE_TX_FLAG) + return -1; + + if (txq->tx_ftag_en) + return -1; + + return 0; +} + static inline void vtx1(volatile struct fm10k_tx_desc *txdp, struct rte_mbuf *pkt, uint64_t flags) @@ -794,31 +863,6 @@ fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, return nb_pkts; } -static void __attribute__((cold)) -fm10k_tx_queue_release_mbufs_vec(struct fm10k_tx_queue *txq) -{ - unsigned i; - const uint16_t max_desc = (uint16_t)(txq->nb_desc - 1); - - if (txq->sw_ring == NULL || txq->nb_free == max_desc) - return; - - /* release the used mbufs in sw_ring */ - for (i = txq->next_dd - (txq->rs_thresh - 1); - i != txq->next_free; - i = (i + 1) & max_desc) - rte_pktmbuf_free_seg(txq->sw_ring[i]); - - txq->nb_free = max_desc; - - /* reset tx_entry */ - for (i = 0; i < txq->nb_desc; i++) - txq->sw_ring[i] = NULL; - - rte_free(txq->sw_ring); - txq->sw_ring = NULL; -} - static void __attribute__((cold)) fm10k_reset_tx_queue(struct fm10k_tx_queue *txq) {