vhost: enhance async enqueue for small packets

[dpdk.git] / lib / librte_vhost / vhost.c

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 3235658..efb136e 100644 (file)
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -106,7 +106,7 @@ __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
                return;
 
        /* To make sure guest memory updates are committed before logging */
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        page = addr / VHOST_LOG_PAGE;
        while (page * VHOST_LOG_PAGE < addr + len) {
@@ -144,7 +144,7 @@ __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
        if (unlikely(!dev->log_base))
                return;
 
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        log_base = (unsigned long *)(uintptr_t)dev->log_base;
 
@@ -163,7 +163,7 @@ __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
 #endif
        }
 
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        vq->log_cache_nb_elem = 0;
 }
@@ -190,7 +190,7 @@ vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
                 * No more room for a new log cache entry,
                 * so write the dirty log map directly.
                 */
-               rte_smp_wmb();
+               rte_atomic_thread_fence(__ATOMIC_RELEASE);
                vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 
                return;
@@ -327,17 +327,17 @@ cleanup_device(struct virtio_net *dev, int destroy)
 static void
 vhost_free_async_mem(struct vhost_virtqueue *vq)
 {
-       if (vq->async_pkts_pending)
-               rte_free(vq->async_pkts_pending);
        if (vq->async_pkts_info)
                rte_free(vq->async_pkts_info);
+       if (vq->async_descs_split)
+               rte_free(vq->async_descs_split);
        if (vq->it_pool)
                rte_free(vq->it_pool);
        if (vq->vec_pool)
                rte_free(vq->vec_pool);
 
-       vq->async_pkts_pending = NULL;
        vq->async_pkts_info = NULL;
+       vq->async_descs_split = NULL;
        vq->it_pool = NULL;
        vq->vec_pool = NULL;
 }
@@ -544,6 +544,11 @@ init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
        }
 
        vq = dev->virtqueue[vring_idx];
+       if (!vq) {
+               VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n",
+                               vring_idx);
+               return;
+       }
 
        memset(vq, 0, sizeof(struct vhost_virtqueue));
 
@@ -570,6 +575,12 @@ reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
        }
 
        vq = dev->virtqueue[vring_idx];
+       if (!vq) {
+               VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n",
+                               vring_idx);
+               return;
+       }
+
        callfd = vq->callfd;
        init_vring_queue(dev, vring_idx);
        vq->callfd = callfd;
@@ -579,22 +590,29 @@ int
 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 {
        struct vhost_virtqueue *vq;
+       uint32_t i;
 
-       vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
-       if (vq == NULL) {
-               VHOST_LOG_CONFIG(ERR,
-                       "Failed to allocate memory for vring:%u.\n", vring_idx);
-               return -1;
-       }
+       /* Also allocate holes, if any, up to requested vring index. */
+       for (i = 0; i <= vring_idx; i++) {
+               if (dev->virtqueue[i])
+                       continue;
 
-       dev->virtqueue[vring_idx] = vq;
-       init_vring_queue(dev, vring_idx);
-       rte_spinlock_init(&vq->access_lock);
-       vq->avail_wrap_counter = 1;
-       vq->used_wrap_counter = 1;
-       vq->signalled_used_valid = false;
+               vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
+               if (vq == NULL) {
+                       VHOST_LOG_CONFIG(ERR,
+                               "Failed to allocate memory for vring:%u.\n", i);
+                       return -1;
+               }
+
+               dev->virtqueue[i] = vq;
+               init_vring_queue(dev, i);
+               rte_spinlock_init(&vq->access_lock);
+               vq->avail_wrap_counter = 1;
+               vq->used_wrap_counter = 1;
+               vq->signalled_used_valid = false;
+       }
 
-       dev->nr_vring += 1;
+       dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1);
 
        return 0;
 }
@@ -1079,11 +1097,11 @@ rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
        if (unlikely(idx >= vq->size))
                return -1;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        vq->inflight_split->desc[idx].inflight = 0;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        vq->inflight_split->used_idx = last_used_idx;
        return 0;
@@ -1122,11 +1140,11 @@ rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx,
        if (unlikely(head >= vq->size))
                return -1;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        inflight_info->desc[head].inflight = 0;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        inflight_info->old_free_head = inflight_info->free_head;
        inflight_info->old_used_idx = inflight_info->used_idx;
@@ -1253,7 +1271,12 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id)
        if (!dev)
                return 0;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return 0;
+
        vq = dev->virtqueue[queue_id];
+       if (!vq)
+               return 0;
 
        rte_spinlock_lock(&vq->access_lock);
 
@@ -1307,7 +1330,7 @@ vhost_enable_notify_packed(struct virtio_net *dev,
                        vq->avail_wrap_counter << 15;
        }
 
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        vq->device_event->flags = flags;
        return 0;
@@ -1340,7 +1363,12 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
        if (!dev)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
+       if (!vq)
+               return -1;
 
        rte_spinlock_lock(&vq->access_lock);
 
@@ -1450,6 +1478,9 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id,
        if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
        if (!vq)
                return -1;
@@ -1476,6 +1507,9 @@ int rte_vhost_set_vring_base(int vid, uint16_t queue_id,
        if (!dev)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
        if (!vq)
                return -1;
@@ -1500,15 +1534,23 @@ rte_vhost_get_vring_base_from_inflight(int vid,
                                       uint16_t *last_used_idx)
 {
        struct rte_vhost_inflight_info_packed *inflight_info;
+       struct vhost_virtqueue *vq;
        struct virtio_net *dev = get_device(vid);
 
        if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
+       vq = dev->virtqueue[queue_id];
+       if (!vq)
+               return -1;
+
        if (!vq_is_packed(dev))
                return -1;
 
-       inflight_info = dev->virtqueue[queue_id]->inflight_packed;
+       inflight_info = vq->inflight_packed;
        if (!inflight_info)
                return -1;
 
@@ -1546,6 +1588,9 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,
 
        f.intval = features;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
 
        if (unlikely(vq == NULL || !dev->async_copy))
@@ -1583,9 +1628,6 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,
        node = SOCKET_ID_ANY;
 #endif
 
-       vq->async_pkts_pending = rte_malloc_socket(NULL,
-                       vq->size * sizeof(uintptr_t),
-                       RTE_CACHE_LINE_SIZE, node);
        vq->async_pkts_info = rte_malloc_socket(NULL,
                        vq->size * sizeof(struct async_inflight_info),
                        RTE_CACHE_LINE_SIZE, node);
@@ -1595,7 +1637,10 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,
        vq->vec_pool = rte_malloc_socket(NULL,
                        VHOST_MAX_ASYNC_VEC * sizeof(struct iovec),
                        RTE_CACHE_LINE_SIZE, node);
-       if (!vq->async_pkts_pending || !vq->async_pkts_info ||
+       vq->async_descs_split = rte_malloc_socket(NULL,
+                       vq->size * sizeof(struct vring_used_elem),
+                       RTE_CACHE_LINE_SIZE, node);
+       if (!vq->async_descs_split || !vq->async_pkts_info ||
                !vq->it_pool || !vq->vec_pool) {
                vhost_free_async_mem(vq);
                VHOST_LOG_CONFIG(ERR,
@@ -1627,16 +1672,24 @@ int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id)
        if (dev == NULL)
                return ret;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return ret;
+
        vq = dev->virtqueue[queue_id];
 
        if (vq == NULL)
                return ret;
 
        ret = 0;
-       rte_spinlock_lock(&vq->access_lock);
 
        if (!vq->async_registered)
-               goto out;
+               return ret;
+
+       if (!rte_spinlock_trylock(&vq->access_lock)) {
+               VHOST_LOG_CONFIG(ERR, "Failed to unregister async channel. "
+                       "virt queue busy.\n");
+               return -1;
+       }
 
        if (vq->async_pkts_inflight_n) {
                VHOST_LOG_CONFIG(ERR, "Failed to unregister async channel. "