net/i40e: add packet type metadata in vector Rx

[dpdk.git] / drivers / net / i40e / i40e_rxtx_vec.c

diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec.c
index f7a62a8..e429c02 100644 (file)
--- a/drivers/net/i40e/i40e_rxtx_vec.c
+++ b/drivers/net/i40e/i40e_rxtx_vec.c
@@ -144,23 +144,24 @@ desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
                uint64_t dword;
        } vol;
 
-       /* mask everything except rss and vlan flags
-       *bit2 is for vlan tag, bits 13:12 for rss
-       */
+       /* mask everything except RSS, flow director and VLAN flags
+        * bit2 is for VLAN tag, bit11 for flow director indication
+        * bit13:12 for RSS indication.
+        */
        const __m128i rss_vlan_msk = _mm_set_epi16(
                        0x0000, 0x0000, 0x0000, 0x0000,
-                       0x3004, 0x3004, 0x3004, 0x3004);
+                       0x3804, 0x3804, 0x3804, 0x3804);
 
        /* map rss and vlan type to rss hash and vlan flag */
        const __m128i vlan_flags = _mm_set_epi8(0, 0, 0, 0,
                        0, 0, 0, 0,
-                       0, 0, 0, PKT_RX_VLAN_PKT,
+                       0, 0, 0, PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
                        0, 0, 0, 0);
 
        const __m128i rss_flags = _mm_set_epi8(0, 0, 0, 0,
                        0, 0, 0, 0,
-                       0, 0, 0, 0,
-                       PKT_RX_FDIR, 0, PKT_RX_RSS_HASH, 0);
+                       PKT_RX_RSS_HASH | PKT_RX_FDIR, PKT_RX_RSS_HASH, 0, 0,
+                       0, 0, PKT_RX_FDIR, 0);
 
        vlan0 = _mm_unpackhi_epi16(descs[0], descs[1]);
        vlan1 = _mm_unpackhi_epi16(descs[2], descs[3]);
@@ -169,7 +170,7 @@ desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
        vlan1 = _mm_and_si128(vlan0, rss_vlan_msk);
        vlan0 = _mm_shuffle_epi8(vlan_flags, vlan1);
 
-       rss = _mm_srli_epi16(vlan1, 12);
+       rss = _mm_srli_epi16(vlan1, 11);
        rss = _mm_shuffle_epi8(rss_flags, rss);
 
        vlan0 = _mm_or_si128(vlan0, rss);
@@ -186,6 +187,21 @@ desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
 
 #define PKTLEN_SHIFT     10
 
+static inline void
+desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
+{
+       __m128i ptype0 = _mm_unpackhi_epi64(descs[0], descs[1]);
+       __m128i ptype1 = _mm_unpackhi_epi64(descs[2], descs[3]);
+
+       ptype0 = _mm_srli_epi64(ptype0, 30);
+       ptype1 = _mm_srli_epi64(ptype1, 30);
+
+       rx_pkts[0]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype0, 0));
+       rx_pkts[1]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype0, 8));
+       rx_pkts[2]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype1, 0));
+       rx_pkts[3]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype1, 8));
+}
+
  /*
  * Notice:
  * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
@@ -223,7 +239,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
         */
        rxdp = rxq->rx_ring + rxq->rx_tail;
 
-       _mm_prefetch((const void *)rxdp, _MM_HINT_T0);
+       rte_prefetch0(rxdp);
 
        /* See if we need to rearm the RX queue - gives the prefetch a bit
         * of time to act
@@ -268,7 +284,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
         * D. fill info. from desc to mbuf
         */
 
-       for (pos = 0, nb_pkts_recd = 0; pos < RTE_I40E_VPMD_RX_BURST;
+       for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
                        pos += RTE_I40E_DESCS_PER_LOOP,
                        rxdp += RTE_I40E_DESCS_PER_LOOP) {
                __m128i descs[RTE_I40E_DESCS_PER_LOOP];
@@ -297,10 +313,10 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
 
                if (split_packet) {
-                       rte_prefetch0(&rx_pkts[pos]->cacheline1);
-                       rte_prefetch0(&rx_pkts[pos + 1]->cacheline1);
-                       rte_prefetch0(&rx_pkts[pos + 2]->cacheline1);
-                       rte_prefetch0(&rx_pkts[pos + 3]->cacheline1);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
                }
 
                /* avoid compiler reorder optimization */
@@ -392,6 +408,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                                 pkt_mb2);
                _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
                                 pkt_mb1);
+               desc_to_ptype_v(descs, &rx_pkts[pos]);
                /* C.4 calc avaialbe number of desc */
                var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
                nb_pkts_recd += var;
@@ -691,8 +708,20 @@ i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
                return;
 
        /* free all mbufs that are valid in the ring */
-       for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask)
-               rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+       if (rxq->rxrearm_nb == 0) {
+               for (i = 0; i < rxq->nb_rx_desc; i++) {
+                       if (rxq->sw_ring[i].mbuf != NULL)
+                               rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+               }
+       } else {
+               for (i = rxq->rx_tail;
+                    i != rxq->rxrearm_start;
+                    i = (i + 1) & mask) {
+                       if (rxq->sw_ring[i].mbuf != NULL)
+                               rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+               }
+       }
+
        rxq->rxrearm_nb = rxq->nb_rx_desc;
 
        /* set all entries to NULL */