net/mlx5: add C++ include guard to public header

[dpdk.git] / drivers / net / i40e / i40e_rxtx_vec_neon.c

diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c
index 8f3188e..5074685 100644 (file)
--- a/drivers/net/i40e/i40e_rxtx_vec_neon.c
+++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c
@@ -93,43 +93,43 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
                        0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804};
 
        const uint32x4_t cksum_mask = {
-                       PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
-                       PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
-                       PKT_RX_OUTER_IP_CKSUM_BAD,
-                       PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
-                       PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
-                       PKT_RX_OUTER_IP_CKSUM_BAD,
-                       PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
-                       PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
-                       PKT_RX_OUTER_IP_CKSUM_BAD,
-                       PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
-                       PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
-                       PKT_RX_OUTER_IP_CKSUM_BAD};
+                       RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD |
+                       RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+                       RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+                       RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD |
+                       RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+                       RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+                       RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD |
+                       RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+                       RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+                       RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD |
+                       RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+                       RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD};
 
        /* map rss and vlan type to rss hash and vlan flag */
        const uint8x16_t vlan_flags = {
                        0, 0, 0, 0,
-                       PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0, 0, 0,
+                       RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED, 0, 0, 0,
                        0, 0, 0, 0,
                        0, 0, 0, 0};
 
        const uint8x16_t rss_flags = {
-                       0, PKT_RX_FDIR, 0, 0,
-                       0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR,
+                       0, RTE_MBUF_F_RX_FDIR, 0, 0,
+                       0, 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_FDIR,
                        0, 0, 0, 0,
                        0, 0, 0, 0};
 
        const uint8x16_t l3_l4e_flags = {
-                       (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1,
-                       PKT_RX_IP_CKSUM_BAD >> 1,
-                       (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1,
-                       (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1,
-                       (PKT_RX_IP_CKSUM_GOOD | PKT_RX_OUTER_IP_CKSUM_BAD) >> 1,
-                       (PKT_RX_OUTER_IP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1,
-                       (PKT_RX_IP_CKSUM_GOOD | PKT_RX_OUTER_IP_CKSUM_BAD |
-                        PKT_RX_L4_CKSUM_BAD) >> 1,
-                       (PKT_RX_OUTER_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |
-                        PKT_RX_IP_CKSUM_BAD) >> 1,
+                       (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD) >> 1,
+                       RTE_MBUF_F_RX_IP_CKSUM_BAD >> 1,
+                       (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD) >> 1,
+                       (RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
+                       (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD) >> 1,
+                       (RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
+                       (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
+                        RTE_MBUF_F_RX_L4_CKSUM_BAD) >> 1,
+                       (RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+                        RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
                        0, 0, 0, 0, 0, 0, 0, 0};
 
        vlan0 = vzipq_u32(vreinterpretq_u32_u64(descs[0]),
@@ -151,7 +151,7 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
                                              vreinterpretq_u8_u32(l3_l4e)));
        /* then we shift left 1 bit */
        l3_l4e = vshlq_n_u32(l3_l4e, 1);
-       /* we need to mask out the reduntant bits */
+       /* we need to mask out the redundant bits */
        l3_l4e = vandq_u32(l3_l4e, cksum_mask);
 
        vlan0 = vorrq_u32(vlan0, rss);
@@ -286,6 +286,14 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
                descs[1] =  vld1q_u64((uint64_t *)(rxdp + 1));
                descs[0] =  vld1q_u64((uint64_t *)(rxdp));
 
+               /* Use acquire fence to order loads of descriptor qwords */
+               rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
+               /* A.2 reload qword0 to make it ordered after qword1 load */
+               descs[3] = vld1q_lane_u64((uint64_t *)(rxdp + 3), descs[3], 0);
+               descs[2] = vld1q_lane_u64((uint64_t *)(rxdp + 2), descs[2], 0);
+               descs[1] = vld1q_lane_u64((uint64_t *)(rxdp + 1), descs[1], 0);
+               descs[0] = vld1q_lane_u64((uint64_t *)(rxdp), descs[0], 0);
+
                /* B.1 load 4 mbuf point */
                mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
                mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
@@ -301,18 +309,6 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
                        rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
                }
 
-               /* C.1 4=>2 filter staterr info only */
-               sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]),
-                                      vreinterpretq_u16_u64(descs[3]));
-               sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]),
-                                      vreinterpretq_u16_u64(descs[2]));
-
-               /* C.2 get 4 pkts staterr value  */
-               staterr = vzipq_u16(sterr_tmp1.val[1],
-                                   sterr_tmp2.val[1]).val[0];
-
-               desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
-
                /* pkts shift the pktlen field to be 16-bit aligned*/
                uint32x4_t len3 = vshlq_u32(vreinterpretq_u32_u64(descs[3]),
                                            len_shl);
@@ -367,10 +363,22 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
 
                desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 
+               desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
+
                if (likely(pos + RTE_I40E_DESCS_PER_LOOP < nb_pkts)) {
                        rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
                }
 
+               /* C.1 4=>2 filter staterr info only */
+               sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]),
+                                      vreinterpretq_u16_u64(descs[3]));
+               sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]),
+                                      vreinterpretq_u16_u64(descs[2]));
+
+               /* C.2 get 4 pkts staterr value  */
+               staterr = vzipq_u16(sterr_tmp1.val[1],
+                                   sterr_tmp2.val[1]).val[0];
+
                /* C* extract and record EOP bit */
                if (split_packet) {
                        uint8x16_t eop_shuf_mask = {
@@ -408,7 +416,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
                                            I40E_UINT16_BIT - 1));
                stat = ~vgetq_lane_u64(vreinterpretq_u64_u16(staterr), 0);
 
-               /* C.4 calc avaialbe number of desc */
+               /* C.4 calc available number of desc */
                if (unlikely(stat == 0)) {
                        nb_pkts_recd += RTE_I40E_DESCS_PER_LOOP;
                } else {