eal/windows: add missing SPDX license tag

[dpdk.git] / lib / librte_vhost / vhost.c

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 6068c38..52ab93d 100644 (file)
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -26,6 +26,7 @@
 #include "vhost_user.h"
 
 struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
+pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER;
 
 /* Called with iotlb_lock read-locked */
 uint64_t
@@ -106,7 +107,7 @@ __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
                return;
 
        /* To make sure guest memory updates are committed before logging */
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        page = addr / VHOST_LOG_PAGE;
        while (page * VHOST_LOG_PAGE < addr + len) {
@@ -144,7 +145,7 @@ __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
        if (unlikely(!dev->log_base))
                return;
 
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        log_base = (unsigned long *)(uintptr_t)dev->log_base;
 
@@ -163,7 +164,7 @@ __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
 #endif
        }
 
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        vq->log_cache_nb_elem = 0;
 }
@@ -190,7 +191,7 @@ vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
                 * No more room for a new log cache entry,
                 * so write the dirty log map directly.
                 */
-               rte_smp_wmb();
+               rte_atomic_thread_fence(__ATOMIC_RELEASE);
                vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 
                return;
@@ -327,17 +328,17 @@ cleanup_device(struct virtio_net *dev, int destroy)
 static void
 vhost_free_async_mem(struct vhost_virtqueue *vq)
 {
-       if (vq->async_pkts_pending)
-               rte_free(vq->async_pkts_pending);
        if (vq->async_pkts_info)
                rte_free(vq->async_pkts_info);
+       if (vq->async_descs_split)
+               rte_free(vq->async_descs_split);
        if (vq->it_pool)
                rte_free(vq->it_pool);
        if (vq->vec_pool)
                rte_free(vq->vec_pool);
 
-       vq->async_pkts_pending = NULL;
        vq->async_pkts_info = NULL;
+       vq->async_descs_split = NULL;
        vq->it_pool = NULL;
        vq->vec_pool = NULL;
 }
@@ -544,6 +545,11 @@ init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
        }
 
        vq = dev->virtqueue[vring_idx];
+       if (!vq) {
+               VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n",
+                               vring_idx);
+               return;
+       }
 
        memset(vq, 0, sizeof(struct vhost_virtqueue));
 
@@ -570,6 +576,12 @@ reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
        }
 
        vq = dev->virtqueue[vring_idx];
+       if (!vq) {
+               VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n",
+                               vring_idx);
+               return;
+       }
+
        callfd = vq->callfd;
        init_vring_queue(dev, vring_idx);
        vq->callfd = callfd;
@@ -579,22 +591,29 @@ int
 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 {
        struct vhost_virtqueue *vq;
+       uint32_t i;
 
-       vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
-       if (vq == NULL) {
-               VHOST_LOG_CONFIG(ERR,
-                       "Failed to allocate memory for vring:%u.\n", vring_idx);
-               return -1;
-       }
+       /* Also allocate holes, if any, up to requested vring index. */
+       for (i = 0; i <= vring_idx; i++) {
+               if (dev->virtqueue[i])
+                       continue;
 
-       dev->virtqueue[vring_idx] = vq;
-       init_vring_queue(dev, vring_idx);
-       rte_spinlock_init(&vq->access_lock);
-       vq->avail_wrap_counter = 1;
-       vq->used_wrap_counter = 1;
-       vq->signalled_used_valid = false;
+               vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
+               if (vq == NULL) {
+                       VHOST_LOG_CONFIG(ERR,
+                               "Failed to allocate memory for vring:%u.\n", i);
+                       return -1;
+               }
+
+               dev->virtqueue[i] = vq;
+               init_vring_queue(dev, i);
+               rte_spinlock_init(&vq->access_lock);
+               vq->avail_wrap_counter = 1;
+               vq->used_wrap_counter = 1;
+               vq->signalled_used_valid = false;
+       }
 
-       dev->nr_vring += 1;
+       dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1);
 
        return 0;
 }
@@ -627,6 +646,7 @@ vhost_new_device(void)
        struct virtio_net *dev;
        int i;
 
+       pthread_mutex_lock(&vhost_dev_lock);
        for (i = 0; i < MAX_VHOST_DEVICE; i++) {
                if (vhost_devices[i] == NULL)
                        break;
@@ -635,6 +655,7 @@ vhost_new_device(void)
        if (i == MAX_VHOST_DEVICE) {
                VHOST_LOG_CONFIG(ERR,
                        "Failed to find a free slot for new device.\n");
+               pthread_mutex_unlock(&vhost_dev_lock);
                return -1;
        }
 
@@ -642,10 +663,13 @@ vhost_new_device(void)
        if (dev == NULL) {
                VHOST_LOG_CONFIG(ERR,
                        "Failed to allocate memory for new dev.\n");
+               pthread_mutex_unlock(&vhost_dev_lock);
                return -1;
        }
 
        vhost_devices[i] = dev;
+       pthread_mutex_unlock(&vhost_dev_lock);
+
        dev->vid = i;
        dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET;
        dev->slave_req_fd = -1;
@@ -1079,11 +1103,11 @@ rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
        if (unlikely(idx >= vq->size))
                return -1;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        vq->inflight_split->desc[idx].inflight = 0;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        vq->inflight_split->used_idx = last_used_idx;
        return 0;
@@ -1122,11 +1146,11 @@ rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx,
        if (unlikely(head >= vq->size))
                return -1;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        inflight_info->desc[head].inflight = 0;
 
-       rte_smp_mb();
+       rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
 
        inflight_info->old_free_head = inflight_info->free_head;
        inflight_info->old_used_idx = inflight_info->used_idx;
@@ -1253,7 +1277,12 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id)
        if (!dev)
                return 0;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return 0;
+
        vq = dev->virtqueue[queue_id];
+       if (!vq)
+               return 0;
 
        rte_spinlock_lock(&vq->access_lock);
 
@@ -1307,7 +1336,7 @@ vhost_enable_notify_packed(struct virtio_net *dev,
                        vq->avail_wrap_counter << 15;
        }
 
-       rte_smp_wmb();
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
        vq->device_event->flags = flags;
        return 0;
@@ -1340,7 +1369,12 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
        if (!dev)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
+       if (!vq)
+               return -1;
 
        rte_spinlock_lock(&vq->access_lock);
 
@@ -1450,6 +1484,9 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id,
        if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
        if (!vq)
                return -1;
@@ -1476,6 +1513,9 @@ int rte_vhost_set_vring_base(int vid, uint16_t queue_id,
        if (!dev)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
        if (!vq)
                return -1;
@@ -1500,15 +1540,23 @@ rte_vhost_get_vring_base_from_inflight(int vid,
                                       uint16_t *last_used_idx)
 {
        struct rte_vhost_inflight_info_packed *inflight_info;
+       struct vhost_virtqueue *vq;
        struct virtio_net *dev = get_device(vid);
 
        if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
                return -1;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
+       vq = dev->virtqueue[queue_id];
+       if (!vq)
+               return -1;
+
        if (!vq_is_packed(dev))
                return -1;
 
-       inflight_info = dev->virtqueue[queue_id]->inflight_packed;
+       inflight_info = vq->inflight_packed;
        if (!inflight_info)
                return -1;
 
@@ -1546,6 +1594,9 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,
 
        f.intval = features;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return -1;
+
        vq = dev->virtqueue[queue_id];
 
        if (unlikely(vq == NULL || !dev->async_copy))
@@ -1583,9 +1634,6 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,
        node = SOCKET_ID_ANY;
 #endif
 
-       vq->async_pkts_pending = rte_malloc_socket(NULL,
-                       vq->size * sizeof(uintptr_t),
-                       RTE_CACHE_LINE_SIZE, node);
        vq->async_pkts_info = rte_malloc_socket(NULL,
                        vq->size * sizeof(struct async_inflight_info),
                        RTE_CACHE_LINE_SIZE, node);
@@ -1595,7 +1643,10 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,
        vq->vec_pool = rte_malloc_socket(NULL,
                        VHOST_MAX_ASYNC_VEC * sizeof(struct iovec),
                        RTE_CACHE_LINE_SIZE, node);
-       if (!vq->async_pkts_pending || !vq->async_pkts_info ||
+       vq->async_descs_split = rte_malloc_socket(NULL,
+                       vq->size * sizeof(struct vring_used_elem),
+                       RTE_CACHE_LINE_SIZE, node);
+       if (!vq->async_descs_split || !vq->async_pkts_info ||
                !vq->it_pool || !vq->vec_pool) {
                vhost_free_async_mem(vq);
                VHOST_LOG_CONFIG(ERR,
@@ -1627,6 +1678,9 @@ int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id)
        if (dev == NULL)
                return ret;
 
+       if (queue_id >= VHOST_MAX_VRING)
+               return ret;
+
        vq = dev->virtqueue[queue_id];
 
        if (vq == NULL)