X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost.c;h=b83cf639eb8bace1a6bd7afb6e4efdf4a4196b1a;hb=96d1d898dcbc4319d8416b222aa6ba081e605823;hp=0266318440f00fa85baef093da36ca7a273b1846;hpb=4f37df14c405b754b5e971c75f4f67f4bb5bfdde;p=dpdk.git diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 0266318440..b83cf639eb 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -27,9 +27,6 @@ struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; -int vhost_config_log_level; -int vhost_data_log_level; - /* Called with iotlb_lock read-locked */ uint64_t __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, @@ -327,13 +324,33 @@ cleanup_device(struct virtio_net *dev, int destroy) } } +static void +vhost_free_async_mem(struct vhost_virtqueue *vq) +{ + if (vq->async_pkts_pending) + rte_free(vq->async_pkts_pending); + if (vq->async_pkts_info) + rte_free(vq->async_pkts_info); + if (vq->it_pool) + rte_free(vq->it_pool); + if (vq->vec_pool) + rte_free(vq->vec_pool); + + vq->async_pkts_pending = NULL; + vq->async_pkts_info = NULL; + vq->it_pool = NULL; + vq->vec_pool = NULL; +} + void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) { if (vq_is_packed(dev)) rte_free(vq->shadow_used_packed); - else + else { rte_free(vq->shadow_used_split); + vhost_free_async_mem(vq); + } rte_free(vq->batch_copy_elems); rte_mempool_free(vq->iotlb_pool); rte_free(vq); @@ -527,17 +544,21 @@ init_vring_queue(struct virtio_net *dev, uint32_t vring_idx) } vq = dev->virtqueue[vring_idx]; + if (!vq) { + VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n", + vring_idx); + return; + } memset(vq, 0, sizeof(struct vhost_virtqueue)); vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; + vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF; vhost_user_iotlb_init(dev, vring_idx); /* Backends are set to -1 indicating an inactive device. */ vq->backend = -1; - - TAILQ_INIT(&vq->zmbuf_list); } static void @@ -554,6 +575,12 @@ reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx) } vq = dev->virtqueue[vring_idx]; + if (!vq) { + VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n", + vring_idx); + return; + } + callfd = vq->callfd; init_vring_queue(dev, vring_idx); vq->callfd = callfd; @@ -563,22 +590,29 @@ int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) { struct vhost_virtqueue *vq; + uint32_t i; - vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); - if (vq == NULL) { - VHOST_LOG_CONFIG(ERR, - "Failed to allocate memory for vring:%u.\n", vring_idx); - return -1; - } + /* Also allocate holes, if any, up to requested vring index. */ + for (i = 0; i <= vring_idx; i++) { + if (dev->virtqueue[i]) + continue; - dev->virtqueue[vring_idx] = vq; - init_vring_queue(dev, vring_idx); - rte_spinlock_init(&vq->access_lock); - vq->avail_wrap_counter = 1; - vq->used_wrap_counter = 1; - vq->signalled_used_valid = false; + vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); + if (vq == NULL) { + VHOST_LOG_CONFIG(ERR, + "Failed to allocate memory for vring:%u.\n", i); + return -1; + } + + dev->virtqueue[i] = vq; + init_vring_queue(dev, i); + rte_spinlock_init(&vq->access_lock); + vq->avail_wrap_counter = 1; + vq->used_wrap_counter = 1; + vq->signalled_used_valid = false; + } - dev->nr_vring += 1; + dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1); return 0; } @@ -633,7 +667,6 @@ vhost_new_device(void) dev->vid = i; dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; dev->slave_req_fd = -1; - dev->vdpa_dev_id = -1; dev->postcopy_ufd = -1; rte_spinlock_init(&dev->slave_req_lock); @@ -644,12 +677,10 @@ void vhost_destroy_device_notify(struct virtio_net *dev) { struct rte_vdpa_device *vdpa_dev; - int did; if (dev->flags & VIRTIO_DEV_RUNNING) { - did = dev->vdpa_dev_id; - vdpa_dev = rte_vdpa_get_device(did); - if (vdpa_dev && vdpa_dev->ops->dev_close) + vdpa_dev = dev->vdpa_dev; + if (vdpa_dev) vdpa_dev->ops->dev_close(dev->vid); dev->flags &= ~VIRTIO_DEV_RUNNING; dev->notify_ops->destroy_device(dev->vid); @@ -677,17 +708,14 @@ vhost_destroy_device(int vid) } void -vhost_attach_vdpa_device(int vid, int did) +vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev) { struct virtio_net *dev = get_device(vid); if (dev == NULL) return; - if (rte_vdpa_get_device(did) == NULL) - return; - - dev->vdpa_dev_id = did; + dev->vdpa_dev = vdpa_dev; } void @@ -707,17 +735,6 @@ vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) dev->ifname[sizeof(dev->ifname) - 1] = '\0'; } -void -vhost_enable_dequeue_zero_copy(int vid) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return; - - dev->dequeue_zero_copy = 1; -} - void vhost_set_builtin_virtio_net(int vid, bool enable) { @@ -1254,7 +1271,12 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id) if (!dev) return 0; + if (queue_id >= VHOST_MAX_VRING) + return 0; + vq = dev->virtqueue[queue_id]; + if (!vq) + return 0; rte_spinlock_lock(&vq->access_lock); @@ -1314,6 +1336,23 @@ vhost_enable_notify_packed(struct virtio_net *dev, return 0; } +int +vhost_enable_guest_notification(struct virtio_net *dev, + struct vhost_virtqueue *vq, int enable) +{ + /* + * If the virtqueue is not ready yet, it will be applied + * when it will become ready. + */ + if (!vq->ready) + return 0; + + if (vq_is_packed(dev)) + return vhost_enable_notify_packed(dev, vq, enable); + else + return vhost_enable_notify_split(dev, vq, enable); +} + int rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) { @@ -1324,14 +1363,17 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) if (!dev) return -1; + if (queue_id >= VHOST_MAX_VRING) + return -1; + vq = dev->virtqueue[queue_id]; + if (!vq) + return -1; rte_spinlock_lock(&vq->access_lock); - if (vq_is_packed(dev)) - ret = vhost_enable_notify_packed(dev, vq, enable); - else - ret = vhost_enable_notify_split(dev, vq, enable); + vq->notif_enable = enable; + ret = vhost_enable_guest_notification(dev, vq, enable); rte_spinlock_unlock(&vq->access_lock); @@ -1402,14 +1444,15 @@ out: return ret; } -int rte_vhost_get_vdpa_device_id(int vid) +struct rte_vdpa_device * +rte_vhost_get_vdpa_device(int vid) { struct virtio_net *dev = get_device(vid); if (dev == NULL) - return -1; + return NULL; - return dev->vdpa_dev_id; + return dev->vdpa_dev; } int rte_vhost_get_log_base(int vid, uint64_t *log_base, @@ -1435,6 +1478,9 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id, if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) return -1; + if (queue_id >= VHOST_MAX_VRING) + return -1; + vq = dev->virtqueue[queue_id]; if (!vq) return -1; @@ -1461,6 +1507,9 @@ int rte_vhost_set_vring_base(int vid, uint16_t queue_id, if (!dev) return -1; + if (queue_id >= VHOST_MAX_VRING) + return -1; + vq = dev->virtqueue[queue_id]; if (!vq) return -1; @@ -1485,15 +1534,23 @@ rte_vhost_get_vring_base_from_inflight(int vid, uint16_t *last_used_idx) { struct rte_vhost_inflight_info_packed *inflight_info; + struct vhost_virtqueue *vq; struct virtio_net *dev = get_device(vid); if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) return -1; + if (queue_id >= VHOST_MAX_VRING) + return -1; + + vq = dev->virtqueue[queue_id]; + if (!vq) + return -1; + if (!vq_is_packed(dev)) return -1; - inflight_info = dev->virtqueue[queue_id]->inflight_packed; + inflight_info = vq->inflight_packed; if (!inflight_info) return -1; @@ -1517,13 +1574,141 @@ int rte_vhost_extern_callback_register(int vid, return 0; } -RTE_INIT(vhost_log_init) +int rte_vhost_async_channel_register(int vid, uint16_t queue_id, + uint32_t features, + struct rte_vhost_async_channel_ops *ops) { - vhost_config_log_level = rte_log_register("lib.vhost.config"); - if (vhost_config_log_level >= 0) - rte_log_set_level(vhost_config_log_level, RTE_LOG_INFO); + struct vhost_virtqueue *vq; + struct virtio_net *dev = get_device(vid); + struct rte_vhost_async_features f; + int node; + + if (dev == NULL || ops == NULL) + return -1; + + f.intval = features; + + if (queue_id >= VHOST_MAX_VRING) + return -1; + + vq = dev->virtqueue[queue_id]; + + if (unlikely(vq == NULL || !dev->async_copy)) + return -1; + + /* packed queue is not supported */ + if (unlikely(vq_is_packed(dev) || !f.async_inorder)) { + VHOST_LOG_CONFIG(ERR, + "async copy is not supported on packed queue or non-inorder mode " + "(vid %d, qid: %d)\n", vid, queue_id); + return -1; + } + + if (unlikely(ops->check_completed_copies == NULL || + ops->transfer_data == NULL)) + return -1; + + rte_spinlock_lock(&vq->access_lock); + + if (unlikely(vq->async_registered)) { + VHOST_LOG_CONFIG(ERR, + "async register failed: channel already registered " + "(vid %d, qid: %d)\n", vid, queue_id); + goto reg_out; + } + +#ifdef RTE_LIBRTE_VHOST_NUMA + if (get_mempolicy(&node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) { + VHOST_LOG_CONFIG(ERR, + "unable to get numa information in async register. " + "allocating async buffer memory on the caller thread node\n"); + node = SOCKET_ID_ANY; + } +#else + node = SOCKET_ID_ANY; +#endif + + vq->async_pkts_pending = rte_malloc_socket(NULL, + vq->size * sizeof(uintptr_t), + RTE_CACHE_LINE_SIZE, node); + vq->async_pkts_info = rte_malloc_socket(NULL, + vq->size * sizeof(struct async_inflight_info), + RTE_CACHE_LINE_SIZE, node); + vq->it_pool = rte_malloc_socket(NULL, + VHOST_MAX_ASYNC_IT * sizeof(struct rte_vhost_iov_iter), + RTE_CACHE_LINE_SIZE, node); + vq->vec_pool = rte_malloc_socket(NULL, + VHOST_MAX_ASYNC_VEC * sizeof(struct iovec), + RTE_CACHE_LINE_SIZE, node); + if (!vq->async_pkts_pending || !vq->async_pkts_info || + !vq->it_pool || !vq->vec_pool) { + vhost_free_async_mem(vq); + VHOST_LOG_CONFIG(ERR, + "async register failed: cannot allocate memory for vq data " + "(vid %d, qid: %d)\n", vid, queue_id); + goto reg_out; + } - vhost_data_log_level = rte_log_register("lib.vhost.data"); - if (vhost_data_log_level >= 0) - rte_log_set_level(vhost_data_log_level, RTE_LOG_WARNING); + vq->async_ops.check_completed_copies = ops->check_completed_copies; + vq->async_ops.transfer_data = ops->transfer_data; + + vq->async_inorder = f.async_inorder; + vq->async_threshold = f.async_threshold; + + vq->async_registered = true; + +reg_out: + rte_spinlock_unlock(&vq->access_lock); + + return 0; +} + +int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) +{ + struct vhost_virtqueue *vq; + struct virtio_net *dev = get_device(vid); + int ret = -1; + + if (dev == NULL) + return ret; + + if (queue_id >= VHOST_MAX_VRING) + return ret; + + vq = dev->virtqueue[queue_id]; + + if (vq == NULL) + return ret; + + ret = 0; + + if (!vq->async_registered) + return ret; + + if (!rte_spinlock_trylock(&vq->access_lock)) { + VHOST_LOG_CONFIG(ERR, "Failed to unregister async channel. " + "virt queue busy.\n"); + return -1; + } + + if (vq->async_pkts_inflight_n) { + VHOST_LOG_CONFIG(ERR, "Failed to unregister async channel. " + "async inflight packets must be completed before unregistration.\n"); + ret = -1; + goto out; + } + + vhost_free_async_mem(vq); + + vq->async_ops.transfer_data = NULL; + vq->async_ops.check_completed_copies = NULL; + vq->async_registered = false; + +out: + rte_spinlock_unlock(&vq->access_lock); + + return ret; } + +RTE_LOG_REGISTER(vhost_config_log_level, lib.vhost.config, INFO); +RTE_LOG_REGISTER(vhost_data_log_level, lib.vhost.data, WARNING);