net/virtio: add NEON based Rx handler

[dpdk.git] / drivers / net / virtio / virtio_rxtx.c

diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 9aba044..9ab441b 100644 (file)
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -50,6 +50,7 @@
 #include <rte_string_fns.h>
 #include <rte_errno.h>
 #include <rte_byteorder.h>
+#include <rte_cpuflags.h>
 
 #include "virtio_logs.h"
 #include "virtio_ethdev.h"
@@ -67,10 +68,6 @@
 #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
        ETH_TXQ_FLAGS_NOOFFLOADS)
 
-#ifdef RTE_MACHINE_CPUFLAG_SSSE3
-static int use_simple_rxtx;
-#endif
-
 static void
 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
 {
@@ -334,6 +331,7 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev)
         */
        uint16_t i;
        uint16_t desc_idx;
+       struct virtio_hw *hw = dev->data->dev_private;
 
        PMD_INIT_FUNC_TRACE();
 
@@ -354,8 +352,7 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev)
                nbufs = 0;
                error = ENOSPC;
 
-#ifdef RTE_MACHINE_CPUFLAG_SSSE3
-               if (use_simple_rxtx) {
+               if (hw->use_simple_rxtx) {
                        for (desc_idx = 0; desc_idx < vq->vq_nentries;
                             desc_idx++) {
                                vq->vq_ring.avail->ring[desc_idx] = desc_idx;
@@ -363,7 +360,7 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev)
                                        VRING_DESC_F_WRITE;
                        }
                }
-#endif
+
                memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf));
                for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST;
                     desc_idx++) {
@@ -379,12 +376,11 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev)
                        /******************************************
                        *         Enqueue allocated buffers        *
                        *******************************************/
-#ifdef RTE_MACHINE_CPUFLAG_SSSE3
-                       if (use_simple_rxtx)
+                       if (hw->use_simple_rxtx)
                                error = virtqueue_enqueue_recv_refill_simple(vq, m);
                        else
-#endif
                                error = virtqueue_enqueue_recv_refill(vq, m);
+
                        if (error) {
                                rte_pktmbuf_free(m);
                                break;
@@ -405,8 +401,7 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev)
                struct virtqueue *vq = txvq->vq;
 
                virtio_dev_vring_start(vq);
-#ifdef RTE_MACHINE_CPUFLAG_SSSE3
-               if (use_simple_rxtx) {
+               if (hw->use_simple_rxtx) {
                        uint16_t mid_idx  = vq->vq_nentries >> 1;
 
                        for (desc_idx = 0; desc_idx < mid_idx; desc_idx++) {
@@ -427,7 +422,7 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev)
                             desc_idx++)
                                vq->vq_ring.avail->ring[desc_idx] = desc_idx;
                }
-#endif
+
                VIRTQUEUE_DUMP(vq);
        }
 }
@@ -457,9 +452,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 
        dev->data->rx_queues[queue_idx] = rxvq;
 
-#ifdef RTE_MACHINE_CPUFLAG_SSSE3
        virtio_rxq_vec_setup(rxvq);
-#endif
 
        return 0;
 }
@@ -468,17 +461,48 @@ void
 virtio_dev_rx_queue_release(void *rxq)
 {
        struct virtnet_rx *rxvq = rxq;
-       struct virtqueue *vq = rxvq->vq;
-       /* rxvq is freed when vq is freed, and as mz should be freed after the
+       struct virtqueue *vq;
+       const struct rte_memzone *mz;
+
+       if (rxvq == NULL)
+               return;
+
+       /*
+        * rxvq is freed when vq is freed, and as mz should be freed after the
         * del_queue, so we reserve the mz pointer first.
         */
-       const struct rte_memzone *mz = rxvq->mz;
+       vq = rxvq->vq;
+       mz = rxvq->mz;
 
-       /* no need to free rxq as vq and rxq are allocated together */
        virtio_dev_queue_release(vq);
        rte_memzone_free(mz);
 }
 
+static void
+virtio_update_rxtx_handler(struct rte_eth_dev *dev,
+                          const struct rte_eth_txconf *tx_conf)
+{
+       uint8_t use_simple_rxtx = 0;
+       struct virtio_hw *hw = dev->data->dev_private;
+
+#if defined RTE_ARCH_X86
+       if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE3))
+               use_simple_rxtx = 1;
+#elif defined RTE_ARCH_ARM64 || defined CONFIG_RTE_ARCH_ARM
+       if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON))
+               use_simple_rxtx = 1;
+#endif
+       /* Use simple rx/tx func if single segment and no offloads */
+       if (use_simple_rxtx &&
+           (tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS &&
+           !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+               PMD_INIT_LOG(INFO, "Using simple rx/tx path");
+               dev->tx_pkt_burst = virtio_xmit_pkts_simple;
+               dev->rx_pkt_burst = virtio_recv_pkts_vec;
+               hw->use_simple_rxtx = use_simple_rxtx;
+       }
+}
+
 /*
  * struct rte_eth_dev *dev: Used to update dev
  * uint16_t nb_desc: Defaults to values read from config space
@@ -494,10 +518,6 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
                        const struct rte_eth_txconf *tx_conf)
 {
        uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
-
-#ifdef RTE_MACHINE_CPUFLAG_SSSE3
-       struct virtio_hw *hw = dev->data->dev_private;
-#endif
        struct virtnet_tx *txvq;
        struct virtqueue *vq;
        uint16_t tx_free_thresh;
@@ -511,16 +531,7 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
                return -EINVAL;
        }
 
-#ifdef RTE_MACHINE_CPUFLAG_SSSE3
-       /* Use simple rx/tx func if single segment and no offloads */
-       if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS &&
-            !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
-               PMD_INIT_LOG(INFO, "Using simple rx/tx path");
-               dev->tx_pkt_burst = virtio_xmit_pkts_simple;
-               dev->rx_pkt_burst = virtio_recv_pkts_vec;
-               use_simple_rxtx = 1;
-       }
-#endif
+       virtio_update_rxtx_handler(dev, tx_conf);
 
        ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
                        nb_desc, socket_id, (void **)&txvq);
@@ -554,12 +565,20 @@ void
 virtio_dev_tx_queue_release(void *txq)
 {
        struct virtnet_tx *txvq = txq;
-       struct virtqueue *vq = txvq->vq;
-       /* txvq is freed when vq is freed, and as mz should be freed after the
+       struct virtqueue *vq;
+       const struct rte_memzone *mz;
+       const struct rte_memzone *hdr_mz;
+
+       if (txvq == NULL)
+               return;
+
+       /*
+        * txvq is freed when vq is freed, and as mz should be freed after the
         * del_queue, so we reserve the mz pointer first.
         */
-       const struct rte_memzone *hdr_mz = txvq->virtio_net_hdr_mz;
-       const struct rte_memzone *mz = txvq->mz;
+       vq = txvq->vq;
+       mz = txvq->mz;
+       hdr_mz = txvq->virtio_net_hdr_mz;
 
        virtio_dev_queue_release(vq);
        rte_memzone_free(mz);