X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=lib%2Flibrte_vhost%2Fvhost_rxtx.c;h=4809d32709a039b5d68794686891d616693f1b69;hb=f161fb6ad5125e8286ea6aaa01e8903dacf21161;hp=0d177b9f88e4636b2ccc967055d37de1694e9322;hpb=830bea8bc4e10445273f6754a2ce9bb6b8bf662b;p=dpdk.git diff --git a/lib/librte_vhost/vhost_rxtx.c b/lib/librte_vhost/vhost_rxtx.c index 0d177b9f88..4809d32709 100644 --- a/lib/librte_vhost/vhost_rxtx.c +++ b/lib/librte_vhost/vhost_rxtx.c @@ -31,45 +31,32 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include +#include -#include -#include -#include -#include -#include -#include +#include +#include +#include -#include "main.h" -#include "virtio-net.h" -#include "vhost-net-cdev.h" +#include "vhost-net.h" -#define MAX_PKT_BURST 32 /* Max burst size for RX/TX */ +#define MAX_PKT_BURST 32 -/* +/** * This function adds buffers to the virtio devices RX virtqueue. Buffers can * be received from the physical port or from another virtio device. A packet - * count is returned to indicate the number of packets that were succesfully + * count is returned to indicate the number of packets that are succesfully * added to the RX queue. This function works when mergeable is disabled. */ static inline uint32_t __attribute__((always_inline)) -virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, uint32_t count) +virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint32_t count) { struct vhost_virtqueue *vq; struct vring_desc *desc; struct rte_mbuf *buff; /* The virtio_hdr is initialised to 0. */ - struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0,0,0,0,0,0},0}; + struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0}; uint64_t buff_addr = 0; uint64_t buff_hdr_addr = 0; uint32_t head[MAX_PKT_BURST], packet_len = 0; @@ -88,7 +75,10 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, vq = dev->virtqueue[VIRTIO_RXQ]; count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count; - /* As many data cores may want access to available buffers, they need to be reserved. */ + /* + * As many data cores may want access to available buffers, + * they need to be reserved. + */ do { res_base_idx = vq->last_used_idx_res; avail_idx = *((volatile uint16_t *)&vq->avail->idx); @@ -103,18 +93,21 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, res_end_idx = res_base_idx + count; /* vq->last_used_idx_res is atomically updated. */ - success = rte_atomic16_cmpset(&vq->last_used_idx_res, res_base_idx, - res_end_idx); + /* TODO: Allow to disable cmpset if no concurrency in application. */ + success = rte_atomic16_cmpset(&vq->last_used_idx_res, + res_base_idx, res_end_idx); } while (unlikely(success == 0)); res_cur_idx = res_base_idx; - LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n", dev->device_fh, res_cur_idx, res_end_idx); + LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n", + dev->device_fh, res_cur_idx, res_end_idx); /* Prefetch available ring to retrieve indexes. */ rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]); /* Retrieve all of the head indexes first to avoid caching issues. */ for (head_idx = 0; head_idx < count; head_idx++) - head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & (vq->size - 1)]; + head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & + (vq->size - 1)]; /*Prefetch descriptor index. */ rte_prefetch0(&vq->desc[head[packet_success]]); @@ -128,7 +121,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */ buff_addr = gpa_to_vva(dev, desc->addr); /* Prefetch buffer address. */ - rte_prefetch0((void*)(uintptr_t)buff_addr); + rte_prefetch0((void *)(uintptr_t)buff_addr); /* Copy virtio_hdr to packet and increment buffer address */ buff_hdr_addr = buff_addr; @@ -150,10 +143,12 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, } /* Update used ring with desc information */ - vq->used->ring[res_cur_idx & (vq->size - 1)].id = head[packet_success]; + vq->used->ring[res_cur_idx & (vq->size - 1)].id = + head[packet_success]; vq->used->ring[res_cur_idx & (vq->size - 1)].len = packet_len; /* Copy mbuf data to buffer */ + /* FIXME for sg mbuf and the case that desc couldn't hold the mbuf data */ rte_memcpy((void *)(uintptr_t)buff_addr, rte_pktmbuf_mtod(buff, const void *), rte_pktmbuf_data_len(buff)); @@ -183,16 +178,18 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, *(volatile uint16_t *)&vq->used->idx += count; vq->last_used_idx = res_end_idx; + /* flush used->idx update before we read avail->flags. */ + rte_mb(); + /* Kick the guest if necessary. */ if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) - eventfd_write((int)vq->kickfd, 1); + eventfd_write((int)vq->callfd, 1); return count; } static inline uint32_t __attribute__((always_inline)) -copy_from_mbuf_to_vring(struct virtio_net *dev, - uint16_t res_base_idx, uint16_t res_end_idx, - struct rte_mbuf *pkt) +copy_from_mbuf_to_vring(struct virtio_net *dev, uint16_t res_base_idx, + uint16_t res_end_idx, struct rte_mbuf *pkt) { uint32_t vec_idx = 0; uint32_t entry_success = 0; @@ -396,14 +393,11 @@ copy_from_mbuf_to_vring(struct virtio_net *dev, } /* - * This function adds buffers to the virtio devices RX virtqueue. Buffers can - * be received from the physical port or from another virtio device. A packet - * count is returned to indicate the number of packets that were succesfully - * added to the RX queue. This function works for mergeable RX. + * This function works for mergeable RX. */ static inline uint32_t __attribute__((always_inline)) -virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, - uint32_t count) +virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint32_t count) { struct vhost_virtqueue *vq; uint32_t pkt_idx = 0, entry_success = 0; @@ -514,17 +508,30 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf * *(volatile uint16_t *)&vq->used->idx += entry_success; vq->last_used_idx = res_end_idx; + /* flush used->idx update before we read avail->flags. */ + rte_mb(); + /* Kick the guest if necessary. */ if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) - eventfd_write((int)vq->kickfd, 1); + eventfd_write((int)vq->callfd, 1); } return count; } -/* This function works for TX packets with mergeable feature enabled. */ -static inline uint16_t __attribute__((always_inline)) -virtio_dev_merge_tx(struct virtio_net *dev, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) +uint16_t +rte_vhost_enqueue_burst(struct virtio_net *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint16_t count) +{ + if (unlikely(dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF))) + return virtio_dev_merge_rx(dev, queue_id, pkts, count); + else + return virtio_dev_rx(dev, queue_id, pkts, count); +} + +uint16_t +rte_vhost_dequeue_burst(struct virtio_net *dev, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) { struct rte_mbuf *m, *prev; struct vhost_virtqueue *vq; @@ -548,7 +555,7 @@ virtio_dev_merge_tx(struct virtio_net *dev, uint16_t queue_id, struct rte_mempoo if (vq->last_used_idx == avail_idx) return 0; - LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_tx()\n", + LOG_DEBUG(VHOST_DATA, "%s (%"PRIu64")\n", __func__, dev->device_fh); /* Prefetch available ring to retrieve head indexes. */ @@ -562,7 +569,7 @@ virtio_dev_merge_tx(struct virtio_net *dev, uint16_t queue_id, struct rte_mempoo free_entries = RTE_MIN(free_entries, MAX_PKT_BURST); LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n", - dev->device_fh, free_entries); + dev->device_fh, free_entries); /* Retrieve all of the head indexes first to avoid caching issues. */ for (i = 0; i < free_entries; i++) head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)]; @@ -608,7 +615,7 @@ virtio_dev_merge_tx(struct virtio_net *dev, uint16_t queue_id, struct rte_mempoo if (unlikely(m == NULL)) { RTE_LOG(ERR, VHOST_DATA, "Failed to allocate memory for mbuf.\n"); - return entry_success; + break; } seg_offset = 0; seg_avail = m->buf_len - RTE_PKTMBUF_HEADROOM; @@ -724,7 +731,6 @@ virtio_dev_merge_tx(struct virtio_net *dev, uint16_t queue_id, struct rte_mempoo vq->used->idx += entry_success; /* Kick guest if required. */ if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) - eventfd_write((int)vq->kickfd, 1); + eventfd_write((int)vq->callfd, 1); return entry_success; - }