X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost.c;h=a8ef55d627fe22029946c4340a23caea6b75393d;hb=4d891f77ddfaf9fbdaaf97a5d6269cffd94b8b9b;hp=534f30cf6b53a9038c616fc8cae42c4a88ce39c9;hpb=37f5e79a271d598c88b85d169f22538a087ab0ad;p=dpdk.git diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 534f30cf6b..a8ef55d627 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -8,6 +8,7 @@ #include #include #ifdef RTE_LIBRTE_VHOST_NUMA +#include #include #endif @@ -68,6 +69,210 @@ __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; } +#define VHOST_LOG_PAGE 4096 + +/* + * Atomically set a bit in memory. + */ +static __rte_always_inline void +vhost_set_bit(unsigned int nr, volatile uint8_t *addr) +{ +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * __sync_ built-ins are deprecated, but __atomic_ ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or_1(addr, (1U << nr)); +#else + __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); +#endif +} + +static __rte_always_inline void +vhost_log_page(uint8_t *log_base, uint64_t page) +{ + vhost_set_bit(page % 8, &log_base[page / 8]); +} + +void +__vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (unlikely(!dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + /* To make sure guest memory updates are committed before logging */ + rte_smp_wmb(); + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + page += 1; + } +} + +void +__vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t len) +{ + uint64_t hva, gpa, map_len; + map_len = len; + + hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); + if (map_len != len) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", + iova); + return; + } + + gpa = hva_to_gpa(dev, hva, len); + if (gpa) + __vhost_log_write(dev, gpa, len); +} + +void +__vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + unsigned long *log_base; + int i; + + if (unlikely(!dev->log_base)) + return; + + rte_smp_wmb(); + + log_base = (unsigned long *)(uintptr_t)dev->log_base; + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * '__sync' builtins are deprecated, but '__atomic' ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or(log_base + elem->offset, elem->val); +#else + __atomic_fetch_or(log_base + elem->offset, elem->val, + __ATOMIC_RELAXED); +#endif + } + + rte_smp_wmb(); + + vq->log_cache_nb_elem = 0; +} + +static __rte_always_inline void +vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t page) +{ + uint32_t bit_nr = page % (sizeof(unsigned long) << 3); + uint32_t offset = page / (sizeof(unsigned long) << 3); + int i; + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + + if (elem->offset == offset) { + elem->val |= (1UL << bit_nr); + return; + } + } + + if (unlikely(i >= VHOST_LOG_CACHE_NR)) { + /* + * No more room for a new log cache entry, + * so write the dirty log map directly. + */ + rte_smp_wmb(); + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + + return; + } + + vq->log_cache[i].offset = offset; + vq->log_cache[i].val = (1UL << bit_nr); + vq->log_cache_nb_elem++; +} + +void +__vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (unlikely(!dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_cache_page(dev, vq, page); + page += 1; + } +} + +void +__vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t len) +{ + uint64_t hva, gpa, map_len; + map_len = len; + + hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); + if (map_len != len) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", + iova); + return; + } + + gpa = hva_to_gpa(dev, hva, len); + if (gpa) + __vhost_log_cache_write(dev, vq, gpa, len); +} + +void * +vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len) +{ + void *idesc; + uint64_t src, dst; + uint64_t len, remain = desc_len; + + idesc = rte_malloc(__func__, desc_len, 0); + if (unlikely(!idesc)) + return NULL; + + dst = (uint64_t)(uintptr_t)idesc; + + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, desc_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + rte_free(idesc); + return NULL; + } + + rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + desc_addr += len; + } + + return idesc; +} + void cleanup_vq(struct vhost_virtqueue *vq, int destroy) { @@ -77,6 +282,31 @@ cleanup_vq(struct vhost_virtqueue *vq, int destroy) close(vq->kickfd); } +void +cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + if (!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))) + return; + + if (vq_is_packed(dev)) { + if (vq->inflight_packed) + vq->inflight_packed = NULL; + } else { + if (vq->inflight_split) + vq->inflight_split = NULL; + } + + if (vq->resubmit_inflight) { + if (vq->resubmit_inflight->resubmit_list) { + free(vq->resubmit_inflight->resubmit_list); + vq->resubmit_inflight->resubmit_list = NULL; + } + free(vq->resubmit_inflight); + vq->resubmit_inflight = NULL; + } +} + /* * Unmap any memory, close any file descriptors and * free any memory owned by a device. @@ -88,8 +318,10 @@ cleanup_device(struct virtio_net *dev, int destroy) vhost_backend_cleanup(dev); - for (i = 0; i < dev->nr_vring; i++) + for (i = 0; i < dev->nr_vring; i++) { cleanup_vq(dev->virtqueue[i], destroy); + cleanup_vq_inflight(dev, dev->virtqueue[i]); + } } void @@ -163,13 +395,28 @@ vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) req_size = sizeof(struct vring_packed_desc) * vq->size; size = req_size; - vq->desc_packed = - (struct vring_packed_desc *)(uintptr_t)vhost_iova_to_vva(dev, - vq, vq->ring_addrs.desc_user_addr, - &size, VHOST_ACCESS_RW); + vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) + vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, + &size, VHOST_ACCESS_RW); if (!vq->desc_packed || size != req_size) return -1; + req_size = sizeof(struct vring_packed_desc_event); + size = req_size; + vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) + vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, + &size, VHOST_ACCESS_RW); + if (!vq->driver_event || size != req_size) + return -1; + + req_size = sizeof(struct vring_packed_desc_event); + size = req_size; + vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) + vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, + &size, VHOST_ACCESS_RW); + if (!vq->device_event || size != req_size) + return -1; + return 0; } @@ -178,7 +425,7 @@ vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) { if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) - goto out; + return -1; if (vq_is_packed(dev)) { if (vring_translate_packed(dev, vq) < 0) @@ -187,7 +434,6 @@ vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) if (vring_translate_split(dev, vq) < 0) return -1; } -out: vq->access_ok = 1; return 0; @@ -203,6 +449,7 @@ vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq) vq->desc = NULL; vq->avail = NULL; vq->used = NULL; + vq->log_guest_addr = 0; if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) vhost_user_iotlb_wr_unlock(vq); @@ -270,6 +517,7 @@ alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) rte_spinlock_init(&vq->access_lock); vq->avail_wrap_counter = 1; vq->used_wrap_counter = 1; + vq->signalled_used_valid = false; dev->nr_vring += 1; @@ -327,6 +575,7 @@ vhost_new_device(void) dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; dev->slave_req_fd = -1; dev->vdpa_dev_id = -1; + dev->postcopy_ufd = -1; rte_spinlock_init(&dev->slave_req_lock); return i; @@ -382,19 +631,6 @@ vhost_attach_vdpa_device(int vid, int did) dev->vdpa_dev_id = did; } -void -vhost_detach_vdpa_device(int vid) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return; - - vhost_user_host_notifier_ctrl(vid, false); - - dev->vdpa_dev_id = -1; -} - void vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) { @@ -442,7 +678,7 @@ rte_vhost_get_mtu(int vid, uint16_t *mtu) { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL || mtu == NULL) return -ENODEV; if (!(dev->flags & VIRTIO_DEV_READY)) @@ -464,7 +700,7 @@ rte_vhost_get_numa_node(int vid) int numa_node; int ret; - if (dev == NULL) + if (dev == NULL || numa_available() != 0) return -1; ret = get_mempolicy(&numa_node, NULL, 0, dev, @@ -510,7 +746,7 @@ rte_vhost_get_ifname(int vid, char *buf, size_t len) { struct virtio_net *dev = get_device(vid); - if (dev == NULL) + if (dev == NULL || buf == NULL) return -1; len = RTE_MIN(len, sizeof(dev->ifname)); @@ -527,7 +763,7 @@ rte_vhost_get_negotiated_features(int vid, uint64_t *features) struct virtio_net *dev; dev = get_device(vid); - if (!dev) + if (dev == NULL || features == NULL) return -1; *features = dev->features; @@ -542,7 +778,7 @@ rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) size_t size; dev = get_device(vid); - if (!dev) + if (dev == NULL || mem == NULL) return -1; size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); @@ -565,7 +801,7 @@ rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, struct vhost_virtqueue *vq; dev = get_device(vid); - if (!dev) + if (dev == NULL || vring == NULL) return -1; if (vring_idx >= VHOST_MAX_VRING) @@ -587,6 +823,314 @@ rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, return 0; } +int +rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx, + struct rte_vhost_ring_inflight *vring) +{ + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (unlikely(!dev)) + return -1; + + if (vring_idx >= VHOST_MAX_VRING) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (unlikely(!vq)) + return -1; + + if (vq_is_packed(dev)) { + if (unlikely(!vq->inflight_packed)) + return -1; + + vring->inflight_packed = vq->inflight_packed; + } else { + if (unlikely(!vq->inflight_split)) + return -1; + + vring->inflight_split = vq->inflight_split; + } + + vring->resubmit_inflight = vq->resubmit_inflight; + + return 0; +} + +int +rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx, + uint16_t idx) +{ + struct vhost_virtqueue *vq; + struct virtio_net *dev; + + dev = get_device(vid); + if (unlikely(!dev)) + return -1; + + if (unlikely(!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) + return 0; + + if (unlikely(vq_is_packed(dev))) + return -1; + + if (unlikely(vring_idx >= VHOST_MAX_VRING)) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (unlikely(!vq)) + return -1; + + if (unlikely(!vq->inflight_split)) + return -1; + + if (unlikely(idx >= vq->size)) + return -1; + + vq->inflight_split->desc[idx].counter = vq->global_counter++; + vq->inflight_split->desc[idx].inflight = 1; + return 0; +} + +int +rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx, + uint16_t head, uint16_t last, + uint16_t *inflight_entry) +{ + struct rte_vhost_inflight_info_packed *inflight_info; + struct virtio_net *dev; + struct vhost_virtqueue *vq; + struct vring_packed_desc *desc; + uint16_t old_free_head, free_head; + + dev = get_device(vid); + if (unlikely(!dev)) + return -1; + + if (unlikely(!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) + return 0; + + if (unlikely(!vq_is_packed(dev))) + return -1; + + if (unlikely(vring_idx >= VHOST_MAX_VRING)) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (unlikely(!vq)) + return -1; + + inflight_info = vq->inflight_packed; + if (unlikely(!inflight_info)) + return -1; + + if (unlikely(head >= vq->size)) + return -1; + + desc = vq->desc_packed; + old_free_head = inflight_info->old_free_head; + if (unlikely(old_free_head >= vq->size)) + return -1; + + free_head = old_free_head; + + /* init header descriptor */ + inflight_info->desc[old_free_head].num = 0; + inflight_info->desc[old_free_head].counter = vq->global_counter++; + inflight_info->desc[old_free_head].inflight = 1; + + /* save desc entry in flight entry */ + while (head != ((last + 1) % vq->size)) { + inflight_info->desc[old_free_head].num++; + inflight_info->desc[free_head].addr = desc[head].addr; + inflight_info->desc[free_head].len = desc[head].len; + inflight_info->desc[free_head].flags = desc[head].flags; + inflight_info->desc[free_head].id = desc[head].id; + + inflight_info->desc[old_free_head].last = free_head; + free_head = inflight_info->desc[free_head].next; + inflight_info->free_head = free_head; + head = (head + 1) % vq->size; + } + + inflight_info->old_free_head = free_head; + *inflight_entry = old_free_head; + + return 0; +} + +int +rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx, + uint16_t last_used_idx, uint16_t idx) +{ + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (unlikely(!dev)) + return -1; + + if (unlikely(!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) + return 0; + + if (unlikely(vq_is_packed(dev))) + return -1; + + if (unlikely(vring_idx >= VHOST_MAX_VRING)) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (unlikely(!vq)) + return -1; + + if (unlikely(!vq->inflight_split)) + return -1; + + if (unlikely(idx >= vq->size)) + return -1; + + rte_smp_mb(); + + vq->inflight_split->desc[idx].inflight = 0; + + rte_smp_mb(); + + vq->inflight_split->used_idx = last_used_idx; + return 0; +} + +int +rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx, + uint16_t head) +{ + struct rte_vhost_inflight_info_packed *inflight_info; + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (unlikely(!dev)) + return -1; + + if (unlikely(!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) + return 0; + + if (unlikely(!vq_is_packed(dev))) + return -1; + + if (unlikely(vring_idx >= VHOST_MAX_VRING)) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (unlikely(!vq)) + return -1; + + inflight_info = vq->inflight_packed; + if (unlikely(!inflight_info)) + return -1; + + if (unlikely(head >= vq->size)) + return -1; + + rte_smp_mb(); + + inflight_info->desc[head].inflight = 0; + + rte_smp_mb(); + + inflight_info->old_free_head = inflight_info->free_head; + inflight_info->old_used_idx = inflight_info->used_idx; + inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter; + + return 0; +} + +int +rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx, + uint16_t idx) +{ + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (unlikely(!dev)) + return -1; + + if (unlikely(!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) + return 0; + + if (unlikely(vq_is_packed(dev))) + return -1; + + if (unlikely(vring_idx >= VHOST_MAX_VRING)) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (unlikely(!vq)) + return -1; + + if (unlikely(!vq->inflight_split)) + return -1; + + vq->inflight_split->last_inflight_io = idx; + return 0; +} + +int +rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx, + uint16_t head) +{ + struct rte_vhost_inflight_info_packed *inflight_info; + struct virtio_net *dev; + struct vhost_virtqueue *vq; + uint16_t last; + + dev = get_device(vid); + if (unlikely(!dev)) + return -1; + + if (unlikely(!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) + return 0; + + if (unlikely(!vq_is_packed(dev))) + return -1; + + if (unlikely(vring_idx >= VHOST_MAX_VRING)) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (unlikely(!vq)) + return -1; + + inflight_info = vq->inflight_packed; + if (unlikely(!inflight_info)) + return -1; + + if (unlikely(head >= vq->size)) + return -1; + + last = inflight_info->desc[head].last; + if (unlikely(last >= vq->size)) + return -1; + + inflight_info->desc[last].next = inflight_info->free_head; + inflight_info->free_head = head; + inflight_info->used_idx += inflight_info->desc[head].num; + if (inflight_info->used_idx >= inflight_info->desc_num) { + inflight_info->used_idx -= inflight_info->desc_num; + inflight_info->used_wrap_counter = + !inflight_info->used_wrap_counter; + } + + return 0; +} + int rte_vhost_vring_call(int vid, uint16_t vring_idx) { @@ -604,7 +1148,11 @@ rte_vhost_vring_call(int vid, uint16_t vring_idx) if (!vq) return -1; - vhost_vring_call(dev, vq); + if (vq_is_packed(dev)) + vhost_vring_call_packed(dev, vq); + else + vhost_vring_call_split(dev, vq); + return 0; } @@ -613,32 +1161,94 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id) { struct virtio_net *dev; struct vhost_virtqueue *vq; + uint16_t ret = 0; dev = get_device(vid); if (!dev) return 0; vq = dev->virtqueue[queue_id]; - if (!vq->enabled) + + rte_spinlock_lock(&vq->access_lock); + + if (unlikely(!vq->enabled || vq->avail == NULL)) + goto out; + + ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; + +out: + rte_spinlock_unlock(&vq->access_lock); + return ret; +} + +static inline int +vhost_enable_notify_split(struct virtio_net *dev, + struct vhost_virtqueue *vq, int enable) +{ + if (vq->used == NULL) + return -1; + + if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { + if (enable) + vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; + else + vq->used->flags |= VRING_USED_F_NO_NOTIFY; + } else { + if (enable) + vhost_avail_event(vq) = vq->last_avail_idx; + } + return 0; +} + +static inline int +vhost_enable_notify_packed(struct virtio_net *dev, + struct vhost_virtqueue *vq, int enable) +{ + uint16_t flags; + + if (vq->device_event == NULL) + return -1; + + if (!enable) { + vq->device_event->flags = VRING_EVENT_F_DISABLE; return 0; + } + + flags = VRING_EVENT_F_ENABLE; + if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { + flags = VRING_EVENT_F_DESC; + vq->device_event->off_wrap = vq->last_avail_idx | + vq->avail_wrap_counter << 15; + } + + rte_smp_wmb(); - return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; + vq->device_event->flags = flags; + return 0; } int rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) { struct virtio_net *dev = get_device(vid); + struct vhost_virtqueue *vq; + int ret; if (!dev) return -1; - if (enable) - dev->virtqueue[queue_id]->used->flags &= - ~VRING_USED_F_NO_NOTIFY; + vq = dev->virtqueue[queue_id]; + + rte_spinlock_lock(&vq->access_lock); + + if (vq_is_packed(dev)) + ret = vhost_enable_notify_packed(dev, vq, enable); else - dev->virtqueue[queue_id]->used->flags |= VRING_USED_F_NO_NOTIFY; - return 0; + ret = vhost_enable_notify_split(dev, vq, enable); + + rte_spinlock_unlock(&vq->access_lock); + + return ret; } void @@ -677,6 +1287,7 @@ rte_vhost_rx_queue_count(int vid, uint16_t qid) { struct virtio_net *dev; struct vhost_virtqueue *vq; + uint32_t ret = 0; dev = get_device(vid); if (dev == NULL) @@ -692,10 +1303,16 @@ rte_vhost_rx_queue_count(int vid, uint16_t qid) if (vq == NULL) return 0; + rte_spinlock_lock(&vq->access_lock); + if (unlikely(vq->enabled == 0 || vq->avail == NULL)) - return 0; + goto out; + + ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; - return *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; +out: + rte_spinlock_unlock(&vq->access_lock); + return ret; } int rte_vhost_get_vdpa_device_id(int vid) @@ -713,15 +1330,8 @@ int rte_vhost_get_log_base(int vid, uint64_t *log_base, { struct virtio_net *dev = get_device(vid); - if (!dev) - return -1; - - if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { - RTE_LOG(ERR, VHOST_DATA, - "(%d) %s: built-in vhost net backend is disabled.\n", - dev->vid, __func__); + if (dev == NULL || log_base == NULL || log_size == NULL) return -1; - } *log_base = dev->log_base; *log_size = dev->log_size; @@ -734,15 +1344,8 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id, { struct virtio_net *dev = get_device(vid); - if (!dev) - return -1; - - if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { - RTE_LOG(ERR, VHOST_DATA, - "(%d) %s: built-in vhost net backend is disabled.\n", - dev->vid, __func__); + if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) return -1; - } *last_avail_idx = dev->virtqueue[queue_id]->last_avail_idx; *last_used_idx = dev->virtqueue[queue_id]->last_used_idx; @@ -750,6 +1353,32 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id, return 0; } +int +rte_vhost_get_vring_base_from_inflight(int vid, + uint16_t queue_id, + uint16_t *last_avail_idx, + uint16_t *last_used_idx) +{ + struct rte_vhost_inflight_info_packed *inflight_info; + struct virtio_net *dev = get_device(vid); + + if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) + return -1; + + if (!vq_is_packed(dev)) + return -1; + + inflight_info = dev->virtqueue[queue_id]->inflight_packed; + if (!inflight_info) + return -1; + + *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) | + inflight_info->old_used_idx; + *last_used_idx = *last_avail_idx; + + return 0; +} + int rte_vhost_set_vring_base(int vid, uint16_t queue_id, uint16_t last_avail_idx, uint16_t last_used_idx) { @@ -758,15 +1387,21 @@ int rte_vhost_set_vring_base(int vid, uint16_t queue_id, if (!dev) return -1; - if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { - RTE_LOG(ERR, VHOST_DATA, - "(%d) %s: built-in vhost net backend is disabled.\n", - dev->vid, __func__); - return -1; - } - dev->virtqueue[queue_id]->last_avail_idx = last_avail_idx; dev->virtqueue[queue_id]->last_used_idx = last_used_idx; return 0; } + +int rte_vhost_extern_callback_register(int vid, + struct rte_vhost_user_extern_ops const * const ops, void *ctx) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL || ops == NULL) + return -1; + + dev->extern_ops = *ops; + dev->extern_data = ctx; + return 0; +}