mbuf: make rearm data address naturally aligned

[dpdk.git] / drivers / net / fm10k / fm10k_rxtx_vec.c

diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
index 1c78725..89cd958 100644 (file)
--- a/drivers/net/fm10k/fm10k_rxtx_vec.c
+++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
@@ -67,6 +67,8 @@ fm10k_reset_tx_queue(struct fm10k_tx_queue *txq);
 #define RXEFLAG_SHIFT     (13)
 /* IPE/L4E flag shift */
 #define L3L4EFLAG_SHIFT     (14)
+/* shift PKT_RX_L4_CKSUM_GOOD into one byte by 1 bit */
+#define CKSUM_SHIFT     (1)
 
 static inline void
 fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
@@ -92,16 +94,23 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
                        0x0000, 0x0000, 0x0000, 0x0000,
                        0x0001, 0x0001, 0x0001, 0x0001);
 
+       /* mask the lower byte of ol_flags */
+       const __m128i ol_flags_msk = _mm_set_epi16(
+                       0x0000, 0x0000, 0x0000, 0x0000,
+                       0x00FF, 0x00FF, 0x00FF, 0x00FF);
+
        const __m128i l3l4cksum_flag = _mm_set_epi8(0, 0, 0, 0,
                        0, 0, 0, 0,
                        0, 0, 0, 0,
-                       PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD,
-                       PKT_RX_IP_CKSUM_BAD, PKT_RX_L4_CKSUM_BAD, 0);
+                       (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD) >> CKSUM_SHIFT,
+                       (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD) >> CKSUM_SHIFT,
+                       (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> CKSUM_SHIFT,
+                       (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> CKSUM_SHIFT);
 
        const __m128i rxe_flag = _mm_set_epi8(0, 0, 0, 0,
                        0, 0, 0, 0,
                        0, 0, 0, 0,
-                       0, 0, PKT_RX_RECIP_ERR, 0);
+                       0, 0, 0, 0);
 
        /* map rss type to rss hash flag */
        const __m128i rss_flags = _mm_set_epi8(0, 0, 0, 0,
@@ -139,6 +148,10 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
        /* Process L4/L3 checksum error flags */
        cksumflag = _mm_srli_epi16(cksumflag, L3L4EFLAG_SHIFT);
        cksumflag = _mm_shuffle_epi8(l3l4cksum_flag, cksumflag);
+
+       /* clean the higher byte and shift back the flag bits */
+       cksumflag = _mm_and_si128(cksumflag, ol_flags_msk);
+       cksumflag = _mm_slli_epi16(cksumflag, CKSUM_SHIFT);
        vtag1 = _mm_or_si128(cksumflag, vtag1);
 
        vol.dword = _mm_cvtsi128_si64(vtag1);
@@ -149,6 +162,9 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
        rx_pkts[3]->ol_flags = vol.e[3];
 }
 
+/* @note: When this function is changed, make corresponding change to
+ * fm10k_dev_supported_ptypes_get().
+ */
 static inline void
 fm10k_desc_to_pktype_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
 {
@@ -231,11 +247,8 @@ fm10k_rx_vec_condition_check(struct rte_eth_dev *dev)
        if (fconf->mode != RTE_FDIR_MODE_NONE)
                return -1;
 
-       /* - no csum error report support
-        * - no header split support
-        */
-       if (rxmode->hw_ip_checksum == 1 ||
-           rxmode->header_split == 1)
+       /* no header split support */
+       if (rxmode->header_split == 1)
                return -1;
 
        return 0;
@@ -311,9 +324,6 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
 
                /* Flush mbuf with pkt template.
                 * Data to be rearmed is 6 bytes long.
-                * Though, RX will overwrite ol_flags that are coming next
-                * anyway. So overwrite whole 8 bytes with one load:
-                * 6 bytes of rearm_data plus first 2 bytes of ol_flags.
                 */
                p0 = (uintptr_t)&mb0->rearm_data;
                *(uint64_t *)p0 = rxq->mbuf_initializer;
@@ -403,7 +413,7 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
         */
        rxdp = rxq->hw_ring + next_dd;
 
-       _mm_prefetch((const void *)rxdp, _MM_HINT_T0);
+       rte_prefetch0(rxdp);
 
        /* See if we need to rearm the RX queue - gives the prefetch a bit
         * of time to act
@@ -465,6 +475,7 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
                /* Read desc statuses backwards to avoid race condition */
                /* A.1 load 4 pkts desc */
                descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
+               rte_compiler_barrier();
 
                /* B.2 copy 2 mbuf point into rx_pkts  */
                _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
@@ -473,8 +484,10 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
                mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]);
 
                descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
+               rte_compiler_barrier();
                /* B.1 load 2 mbuf point */
                descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
+               rte_compiler_barrier();
                descs0[0] = _mm_loadu_si128((__m128i *)(rxdp));
 
                /* B.2 copy 2 mbuf point into rx_pkts  */
@@ -484,10 +497,10 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
                rte_compiler_barrier();
 
                if (split_packet) {
-                       rte_prefetch0(&rx_pkts[pos]->cacheline1);
-                       rte_prefetch0(&rx_pkts[pos + 1]->cacheline1);
-                       rte_prefetch0(&rx_pkts[pos + 2]->cacheline1);
-                       rte_prefetch0(&rx_pkts[pos + 3]->cacheline1);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
                }
 
                /* D.1 pkt 3,4 convert format from desc to pktmbuf */
@@ -603,8 +616,11 @@ fm10k_reassemble_packets(struct fm10k_rx_queue *rxq,
 
                        if (!split_flags[buf_idx]) {
                                /* it's the last packet of the set */
+#ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
                                start->hash = end->hash;
                                start->ol_flags = end->ol_flags;
+                               start->packet_type = end->packet_type;
+#endif
                                pkts[pkt_idx++] = start;
                                start = end = NULL;
                        }
@@ -735,12 +751,12 @@ fm10k_tx_free_bufs(struct fm10k_tx_queue *txq)
         * next_dd - (rs_thresh-1)
         */
        txep = &txq->sw_ring[txq->next_dd - (n - 1)];
-       m = __rte_pktmbuf_prefree_seg(txep[0]);
+       m = rte_pktmbuf_prefree_seg(txep[0]);
        if (likely(m != NULL)) {
                free[0] = m;
                nb_free = 1;
                for (i = 1; i < n; i++) {
-                       m = __rte_pktmbuf_prefree_seg(txep[i]);
+                       m = rte_pktmbuf_prefree_seg(txep[i]);
                        if (likely(m != NULL)) {
                                if (likely(m->pool == free[0]->pool))
                                        free[nb_free++] = m;
@@ -755,7 +771,7 @@ fm10k_tx_free_bufs(struct fm10k_tx_queue *txq)
                rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
        } else {
                for (i = 1; i < n; i++) {
-                       m = __rte_pktmbuf_prefree_seg(txep[i]);
+                       m = rte_pktmbuf_prefree_seg(txep[i]);
                        if (m != NULL)
                                rte_mempool_put(m->pool, m);
                }
@@ -781,8 +797,8 @@ tx_backlog_entry(struct rte_mbuf **txep,
 }
 
 uint16_t
-fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
-                       uint16_t nb_pkts)
+fm10k_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+                          uint16_t nb_pkts)
 {
        struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
        volatile struct fm10k_tx_desc *txdp;