net/vmxnet3: support receive data ring

vmxnet3 driver preallocates buffers for receiving packets and posts the
buffers to the emulation. In order to deliver a received packet to the
guest, the emulation must map buffer(s) and copy the packet into it.

To avoid this memory mapping overhead, this patch introduces the receive
data ring - a set of small sized buffers that are always mapped by
the emulation. If a packet fits into the receive data ring buffer, the
emulation delivers the packet via the receive data ring (which must be
copied by the guest driver), or else the usual receive path is used.

Signed-off-by: Shrikrishna Khare <skhare@vmware.com>
Acked-by: Yong Wang <yongwang@vmware.com>
Acked-by: Jin Heo <heoj@vmware.com>

diff --git a/drivers/net/vmxnet3/base/vmxnet3_defs.h b/drivers/net/vmxnet3/base/vmxnet3_defs.h
index e2018089974330dc661a55c797c4b881a1d8c87c..746d709e57236f80ac48074ddc8a1aa34d34148d 100644 (file)
--- a/drivers/net/vmxnet3/base/vmxnet3_defs.h
+++ b/drivers/net/vmxnet3/base/vmxnet3_defs.h
@@ -408,6 +408,10 @@ typedef union Vmxnet3_GenericDesc {
 #define VMXNET3_TXDATA_DESC_SIZE_ALIGN 64
 #define VMXNET3_TXDATA_DESC_SIZE_MASK  (VMXNET3_TXDATA_DESC_SIZE_ALIGN - 1)
 
+/* Rx Data Ring buffer size must be a multiple of 64 */
+#define VMXNET3_RXDATA_DESC_SIZE_ALIGN 64
+#define VMXNET3_RXDATA_DESC_SIZE_MASK  (VMXNET3_RXDATA_DESC_SIZE_ALIGN - 1)
+
 /* Max ring size */
 #define VMXNET3_TX_RING_MAX_SIZE   4096
 #define VMXNET3_TC_RING_MAX_SIZE   4096
@@ -417,6 +421,8 @@ typedef union Vmxnet3_GenericDesc {
 #define VMXNET3_TXDATA_DESC_MIN_SIZE 128
 #define VMXNET3_TXDATA_DESC_MAX_SIZE 2048
 
+#define VMXNET3_RXDATA_DESC_MAX_SIZE 2048
+
 /* a list of reasons for queue stop */
 
 #define VMXNET3_ERR_NOEOP        0x80000000  /* cannot find the EOP desc of a pkt */
@@ -529,12 +535,14 @@ struct Vmxnet3_RxQueueConf {
    __le64    rxRingBasePA[2];
    __le64    compRingBasePA;
    __le64    ddPA;            /* driver data */
-   __le64    reserved;
+   __le64    rxDataRingBasePA;
    __le32    rxRingSize[2];   /* # of rx desc */
    __le32    compRingSize;    /* # of rx comp desc */
    __le32    ddLen;           /* size of driver data */
    uint8     intrIdx;
-   uint8     _pad[7];
+   uint8     _pad1[1];
+   __le16    rxDataRingDescSize;  /* size of rx data ring buffer */
+   uint8     _pad2[4];
 }
 #include "vmware_pack_end.h"
 Vmxnet3_RxQueueConf;

diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c
index 05b086460ea91ebd9a6a54a53c592d21d82288cf..d92d9ccc7803d2639a79c793c9cea93b8ed542bb 100644 (file)
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c
@@ -341,6 +341,11 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev)
        hw->txdata_desc_size = VMXNET3_VERSION_GE_3(hw) ?
                eth_vmxnet3_txdata_get(hw) : sizeof(struct Vmxnet3_TxDataDesc);
 
+       hw->rxdata_desc_size = VMXNET3_VERSION_GE_3(hw) ?
+               VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
+       RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) ==
+                  hw->rxdata_desc_size);
+
        return 0;
 }
 
@@ -551,6 +556,10 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev)
                rqd->conf.rxRingSize[1]   = rxq->cmd_ring[1].size;
                rqd->conf.compRingSize    = rxq->comp_ring.size;
                rqd->conf.intrIdx         = rxq->comp_ring.intr_idx;
+               if (VMXNET3_VERSION_GE_3(hw)) {
+                       rqd->conf.rxDataRingBasePA = rxq->data_ring.basePA;
+                       rqd->conf.rxDataRingDescSize = rxq->data_desc_size;
+               }
                rqd->status.stopped       = TRUE;
                rqd->status.error         = 0;
                memset(&rqd->stats, 0, sizeof(rqd->stats));

diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.h b/drivers/net/vmxnet3/vmxnet3_ethdev.h
index f1c8bc1f85da89ff8660e77775081b4738b33641..9d00cd101fb9a88597d71efd2e0aa7126bc83252 100644 (file)
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.h
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.h
@@ -104,6 +104,7 @@ struct vmxnet3_hw {
        uint8_t version;
 
        uint16_t txdata_desc_size; /* tx data ring buffer size */
+       uint16_t rxdata_desc_size; /* rx data ring buffer size */
 
        Vmxnet3_TxQueueDesc   *tqd_start;       /* start address of all tx queue desc */
        Vmxnet3_RxQueueDesc   *rqd_start;       /* start address of all rx queue desc */
@@ -157,6 +158,20 @@ vmxnet3_read_addr(volatile void *addr)
 #define VMXNET3_WRITE_BAR1_REG(hw, reg, value) \
        VMXNET3_PCI_REG_WRITE(VMXNET3_PCI_BAR1_REG_ADDR((hw), (reg)), (value))
 
+static inline uint8_t
+vmxnet3_get_ring_idx(struct vmxnet3_hw *hw, uint32 rqID)
+{
+       return (rqID >= hw->num_rx_queues &&
+               rqID < 2 * hw->num_rx_queues) ? 1 : 0;
+}
+
+static inline bool
+vmxnet3_rx_data_ring(struct vmxnet3_hw *hw, uint32 rqID)
+{
+       return (rqID >= 2 * hw->num_rx_queues &&
+               rqID < 3 * hw->num_rx_queues);
+}
+
 /*
  * RX/TX function prototypes
  */

diff --git a/drivers/net/vmxnet3/vmxnet3_ring.h b/drivers/net/vmxnet3/vmxnet3_ring.h
index b27290f3b656c45eee636db4c57bb45c93c99d1f..0ce46c5c961f7f57f3ffe7a72b543a3c10516a4e 100644 (file)
--- a/drivers/net/vmxnet3/vmxnet3_ring.h
+++ b/drivers/net/vmxnet3/vmxnet3_ring.h
@@ -42,6 +42,9 @@
 #define VMXNET3_DEF_TX_RING_SIZE 512
 #define VMXNET3_DEF_RX_RING_SIZE 128
 
+/* Default rx data ring desc size */
+#define VMXNET3_DEF_RXDATA_DESC_SIZE 128
+
 #define VMXNET3_SUCCESS 0
 #define VMXNET3_FAIL   -1
 
@@ -151,13 +154,23 @@ struct vmxnet3_rxq_stats {
        uint64_t                     rx_buf_alloc_failure;
 };
 
+struct vmxnet3_rx_data_ring {
+       uint8_t  *base;
+       uint64_t basePA;
+       uint32_t size;
+};
+
 typedef struct vmxnet3_rx_queue {
        struct rte_mempool          *mp;
        struct vmxnet3_hw           *hw;
        struct vmxnet3_cmd_ring     cmd_ring[VMXNET3_RX_CMDRING_SIZE];
        struct vmxnet3_comp_ring    comp_ring;
+       struct vmxnet3_rx_data_ring data_ring;
+       uint16_t                    data_desc_size;
        uint32_t                    qid1;
        uint32_t                    qid2;
+       /* rqID in RCD for buffer from data ring */
+       uint32_t                    data_ring_qid;
        Vmxnet3_RxQueueDesc         *shared;
        struct rte_mbuf             *start_seg;
        struct rte_mbuf             *last_seg;

diff --git a/drivers/net/vmxnet3/vmxnet3_rxtx.c b/drivers/net/vmxnet3/vmxnet3_rxtx.c
index 705b601ede1ef8206c65b0ebb35550f6c4be6cd4..6649c3f2a3b5a0b3afd55a2a25e1906fe24ebba5 100644 (file)
--- a/drivers/net/vmxnet3/vmxnet3_rxtx.c
+++ b/drivers/net/vmxnet3/vmxnet3_rxtx.c
@@ -254,8 +254,10 @@ vmxnet3_dev_rx_queue_reset(void *rxq)
 {
        int i;
        vmxnet3_rx_queue_t *rq = rxq;
+       struct vmxnet3_hw *hw = rq->hw;
        struct vmxnet3_cmd_ring *ring0, *ring1;
        struct vmxnet3_comp_ring *comp_ring;
+       struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
        int size;
 
        if (rq != NULL) {
@@ -280,6 +282,8 @@ vmxnet3_dev_rx_queue_reset(void *rxq)
 
        size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
        size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
+       if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
+               size += rq->data_desc_size * data_ring->size;
 
        memset(ring0->base, 0, size);
 }
@@ -758,7 +762,7 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
                }
 
                idx = rcd->rxdIdx;
-               ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1);
+               ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
                rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
                RTE_SET_USED(rxd); /* used only for assert when enabled */
                rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
@@ -824,6 +828,15 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
                                goto rcd_done;
                        }
 
+                       if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
+                               uint8_t *rdd = rxq->data_ring.base +
+                                       idx * rxq->data_desc_size;
+
+                               RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
+                               rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
+                                          rdd, rcd->len);
+                       }
+
                        rxq->start_seg = rxm;
                        vmxnet3_rx_offload(rcd, rxm);
                } else {
@@ -1018,6 +1031,7 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
        struct vmxnet3_hw *hw = dev->data->dev_private;
        struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
        struct vmxnet3_comp_ring *comp_ring;
+       struct vmxnet3_rx_data_ring *data_ring;
        int size;
        uint8_t i;
        char mem_name[32];
@@ -1038,11 +1052,14 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
        rxq->hw = hw;
        rxq->qid1 = queue_idx;
        rxq->qid2 = queue_idx + hw->num_rx_queues;
+       rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
+       rxq->data_desc_size = hw->rxdata_desc_size;
        rxq->stopped = TRUE;
 
        ring0 = &rxq->cmd_ring[0];
        ring1 = &rxq->cmd_ring[1];
        comp_ring = &rxq->comp_ring;
+       data_ring = &rxq->data_ring;
 
        /* Rx vmxnet rings length should be between 256-4096 */
        if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
@@ -1058,6 +1075,7 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
        }
 
        comp_ring->size = ring0->size + ring1->size;
+       data_ring->size = ring0->size;
 
        /* Rx vmxnet rings structure initialization */
        ring0->next2fill = 0;
@@ -1071,6 +1089,8 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
 
        size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
        size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
+       if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
+               size += rxq->data_desc_size * data_ring->size;
 
        mz = ring_dma_zone_reserve(dev, "rxdesc", queue_idx, size, socket_id);
        if (mz == NULL) {
@@ -1092,6 +1112,14 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
        comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
                ring1->size;
 
+       /* data_ring initialization */
+       if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
+               data_ring->base =
+                       (uint8_t *)(comp_ring->base + comp_ring->size);
+               data_ring->basePA = comp_ring->basePA +
+                       sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
+       }
+
        /* cmd_ring0-cmd_ring1 buf_info allocation */
        for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {