mbuf: make rearm data address naturally aligned

[dpdk.git] / drivers / net / ixgbe / ixgbe_rxtx_vec_neon.c

diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c
index 9c1d124..ba0b140 100644 (file)
--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c
@@ -85,9 +85,6 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
                /*
                 * Flush mbuf with pkt template.
                 * Data to be rearmed is 6 bytes long.
-                * Though, RX will overwrite ol_flags that are coming next
-                * anyway. So overwrite whole 8 bytes with one load:
-                * 6 bytes of rearm_data plus first 2 bytes of ol_flags.
                 */
                vst1_u8((uint8_t *)&mb0->rearm_data, p);
                paddr = mb0->buf_physaddr + RTE_PKTMBUF_HEADROOM;
@@ -196,7 +193,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
        struct ixgbe_rx_entry *sw_ring;
        uint16_t nb_pkts_recd;
        int pos;
-       uint64_t var;
        uint8x16_t shuf_msk = {
                0xFF, 0xFF,
                0xFF, 0xFF,  /* skip 32 bits pkt_type */
@@ -255,35 +251,33 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                uint64x2_t mbp1, mbp2;
                uint8x16_t staterr;
                uint16x8_t tmp;
+               uint32_t var = 0;
                uint32_t stat;
 
                /* B.1 load 1 mbuf point */
                mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
 
-               /* Read desc statuses backwards to avoid race condition */
-               /* A.1 load 4 pkts desc */
-               descs[3] =  vld1q_u64((uint64_t *)(rxdp + 3));
-               rte_rmb();
-
                /* B.2 copy 2 mbuf point into rx_pkts  */
                vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
 
                /* B.1 load 1 mbuf point */
                mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
 
-               descs[2] =  vld1q_u64((uint64_t *)(rxdp + 2));
-               /* B.1 load 2 mbuf point */
-               descs[1] =  vld1q_u64((uint64_t *)(rxdp + 1));
+               /* A. load 4 pkts descs */
                descs[0] =  vld1q_u64((uint64_t *)(rxdp));
+               descs[1] =  vld1q_u64((uint64_t *)(rxdp + 1));
+               descs[2] =  vld1q_u64((uint64_t *)(rxdp + 2));
+               descs[3] =  vld1q_u64((uint64_t *)(rxdp + 3));
+               rte_smp_rmb();
 
                /* B.2 copy 2 mbuf point into rx_pkts  */
                vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
 
                if (split_packet) {
-                       rte_prefetch_non_temporal(&rx_pkts[pos]->cacheline1);
-                       rte_prefetch_non_temporal(&rx_pkts[pos + 1]->cacheline1);
-                       rte_prefetch_non_temporal(&rx_pkts[pos + 2]->cacheline1);
-                       rte_prefetch_non_temporal(&rx_pkts[pos + 3]->cacheline1);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
+                       rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
                }
 
                /* D.1 pkt 3,4 convert format from desc to pktmbuf */
@@ -333,12 +327,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                        *(int *)split_packet = ~stat & IXGBE_VPMD_DESC_EOP_MASK;
 
                        split_packet += RTE_IXGBE_DESCS_PER_LOOP;
-
-                       /* zero-out next pointers */
-                       rx_pkts[pos]->next = NULL;
-                       rx_pkts[pos + 1]->next = NULL;
-                       rx_pkts[pos + 2]->next = NULL;
-                       rx_pkts[pos + 3]->next = NULL;
                }
 
                rte_prefetch_non_temporal(rxdp + RTE_IXGBE_DESCS_PER_LOOP);
@@ -349,11 +337,19 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                vst1q_u8((uint8_t *)&rx_pkts[pos]->rx_descriptor_fields1,
                         pkt_mb1);
 
+               stat &= IXGBE_VPMD_DESC_DD_MASK;
+
                /* C.4 calc avaialbe number of desc */
-               var =  __builtin_popcount(stat & IXGBE_VPMD_DESC_DD_MASK);
-               nb_pkts_recd += var;
-               if (likely(var != RTE_IXGBE_DESCS_PER_LOOP))
+               if (likely(stat != IXGBE_VPMD_DESC_DD_MASK)) {
+                       while (stat & 0x01) {
+                               ++var;
+                               stat = stat >> 8;
+                       }
+                       nb_pkts_recd += var;
                        break;
+               } else {
+                       nb_pkts_recd += RTE_IXGBE_DESCS_PER_LOOP;
+               }
        }
 
        /* Update our internal tail pointer */
@@ -446,8 +442,8 @@ vtx(volatile union ixgbe_adv_tx_desc *txdp,
 }
 
 uint16_t
-ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
-                      uint16_t nb_pkts)
+ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+                          uint16_t nb_pkts)
 {
        struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
        volatile union ixgbe_adv_tx_desc *txdp;
@@ -556,5 +552,11 @@ ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq)
 int __attribute__((cold))
 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
 {
+       struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
+
+       /* no csum error report support */
+       if (rxmode->hw_ip_checksum == 1)
+               return -1;
+
        return ixgbe_rx_vec_dev_conf_condition_check_default(dev);
 }