X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost.c;h=981837b5dde037eb6a1d9017d5a49eee1f9d3e5d;hb=6b7780bfebe4;hp=534f30cf6b53a9038c616fc8cae42c4a88ce39c9;hpb=37f5e79a271d598c88b85d169f22538a087ab0ad;p=dpdk.git diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 534f30cf6b..981837b5dd 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -8,6 +8,7 @@ #include #include #ifdef RTE_LIBRTE_VHOST_NUMA +#include #include #endif @@ -68,6 +69,170 @@ __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; } +#define VHOST_LOG_PAGE 4096 + +/* + * Atomically set a bit in memory. + */ +static __rte_always_inline void +vhost_set_bit(unsigned int nr, volatile uint8_t *addr) +{ +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * __sync_ built-ins are deprecated, but __atomic_ ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or_1(addr, (1U << nr)); +#else + __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); +#endif +} + +static __rte_always_inline void +vhost_log_page(uint8_t *log_base, uint64_t page) +{ + vhost_set_bit(page % 8, &log_base[page / 8]); +} + +void +__vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (unlikely(!dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + /* To make sure guest memory updates are committed before logging */ + rte_smp_wmb(); + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + page += 1; + } +} + +void +__vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + unsigned long *log_base; + int i; + + if (unlikely(!dev->log_base)) + return; + + rte_smp_wmb(); + + log_base = (unsigned long *)(uintptr_t)dev->log_base; + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * '__sync' builtins are deprecated, but '__atomic' ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or(log_base + elem->offset, elem->val); +#else + __atomic_fetch_or(log_base + elem->offset, elem->val, + __ATOMIC_RELAXED); +#endif + } + + rte_smp_wmb(); + + vq->log_cache_nb_elem = 0; +} + +static __rte_always_inline void +vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t page) +{ + uint32_t bit_nr = page % (sizeof(unsigned long) << 3); + uint32_t offset = page / (sizeof(unsigned long) << 3); + int i; + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + + if (elem->offset == offset) { + elem->val |= (1UL << bit_nr); + return; + } + } + + if (unlikely(i >= VHOST_LOG_CACHE_NR)) { + /* + * No more room for a new log cache entry, + * so write the dirty log map directly. + */ + rte_smp_wmb(); + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + + return; + } + + vq->log_cache[i].offset = offset; + vq->log_cache[i].val = (1UL << bit_nr); + vq->log_cache_nb_elem++; +} + +void +__vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (unlikely(!dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_cache_page(dev, vq, page); + page += 1; + } +} + +void * +vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len) +{ + void *idesc; + uint64_t src, dst; + uint64_t len, remain = desc_len; + + idesc = rte_malloc(__func__, desc_len, 0); + if (unlikely(!idesc)) + return NULL; + + dst = (uint64_t)(uintptr_t)idesc; + + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, desc_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + rte_free(idesc); + return NULL; + } + + rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + desc_addr += len; + } + + return idesc; +} + void cleanup_vq(struct vhost_virtqueue *vq, int destroy) { @@ -163,13 +328,28 @@ vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) req_size = sizeof(struct vring_packed_desc) * vq->size; size = req_size; - vq->desc_packed = - (struct vring_packed_desc *)(uintptr_t)vhost_iova_to_vva(dev, - vq, vq->ring_addrs.desc_user_addr, - &size, VHOST_ACCESS_RW); + vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) + vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, + &size, VHOST_ACCESS_RW); if (!vq->desc_packed || size != req_size) return -1; + req_size = sizeof(struct vring_packed_desc_event); + size = req_size; + vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) + vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, + &size, VHOST_ACCESS_RW); + if (!vq->driver_event || size != req_size) + return -1; + + req_size = sizeof(struct vring_packed_desc_event); + size = req_size; + vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) + vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, + &size, VHOST_ACCESS_RW); + if (!vq->device_event || size != req_size) + return -1; + return 0; } @@ -270,6 +450,7 @@ alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) rte_spinlock_init(&vq->access_lock); vq->avail_wrap_counter = 1; vq->used_wrap_counter = 1; + vq->signalled_used_valid = false; dev->nr_vring += 1; @@ -327,6 +508,7 @@ vhost_new_device(void) dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; dev->slave_req_fd = -1; dev->vdpa_dev_id = -1; + dev->postcopy_ufd = -1; rte_spinlock_init(&dev->slave_req_lock); return i; @@ -382,19 +564,6 @@ vhost_attach_vdpa_device(int vid, int did) dev->vdpa_dev_id = did; } -void -vhost_detach_vdpa_device(int vid) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return; - - vhost_user_host_notifier_ctrl(vid, false); - - dev->vdpa_dev_id = -1; -} - void vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) { @@ -442,7 +611,7 @@ rte_vhost_get_mtu(int vid, uint16_t *mtu) { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL || mtu == NULL) return -ENODEV; if (!(dev->flags & VIRTIO_DEV_READY)) @@ -464,7 +633,7 @@ rte_vhost_get_numa_node(int vid) int numa_node; int ret; - if (dev == NULL) + if (dev == NULL || numa_available() != 0) return -1; ret = get_mempolicy(&numa_node, NULL, 0, dev, @@ -510,7 +679,7 @@ rte_vhost_get_ifname(int vid, char *buf, size_t len) { struct virtio_net *dev = get_device(vid); - if (dev == NULL) + if (dev == NULL || buf == NULL) return -1; len = RTE_MIN(len, sizeof(dev->ifname)); @@ -527,7 +696,7 @@ rte_vhost_get_negotiated_features(int vid, uint64_t *features) struct virtio_net *dev; dev = get_device(vid); - if (!dev) + if (dev == NULL || features == NULL) return -1; *features = dev->features; @@ -542,7 +711,7 @@ rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) size_t size; dev = get_device(vid); - if (!dev) + if (dev == NULL || mem == NULL) return -1; size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); @@ -565,7 +734,7 @@ rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, struct vhost_virtqueue *vq; dev = get_device(vid); - if (!dev) + if (dev == NULL || vring == NULL) return -1; if (vring_idx >= VHOST_MAX_VRING) @@ -604,7 +773,11 @@ rte_vhost_vring_call(int vid, uint16_t vring_idx) if (!vq) return -1; - vhost_vring_call(dev, vq); + if (vq_is_packed(dev)) + vhost_vring_call_packed(dev, vq); + else + vhost_vring_call_split(dev, vq); + return 0; } @@ -625,19 +798,60 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id) return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; } +static inline void +vhost_enable_notify_split(struct virtio_net *dev, + struct vhost_virtqueue *vq, int enable) +{ + if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { + if (enable) + vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; + else + vq->used->flags |= VRING_USED_F_NO_NOTIFY; + } else { + if (enable) + vhost_avail_event(vq) = vq->last_avail_idx; + } +} + +static inline void +vhost_enable_notify_packed(struct virtio_net *dev, + struct vhost_virtqueue *vq, int enable) +{ + uint16_t flags; + + if (!enable) { + vq->device_event->flags = VRING_EVENT_F_DISABLE; + return; + } + + flags = VRING_EVENT_F_ENABLE; + if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { + flags = VRING_EVENT_F_DESC; + vq->device_event->off_wrap = vq->last_avail_idx | + vq->avail_wrap_counter << 15; + } + + rte_smp_wmb(); + + vq->device_event->flags = flags; +} + int rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) { struct virtio_net *dev = get_device(vid); + struct vhost_virtqueue *vq; if (!dev) return -1; - if (enable) - dev->virtqueue[queue_id]->used->flags &= - ~VRING_USED_F_NO_NOTIFY; + vq = dev->virtqueue[queue_id]; + + if (vq_is_packed(dev)) + vhost_enable_notify_packed(dev, vq, enable); else - dev->virtqueue[queue_id]->used->flags |= VRING_USED_F_NO_NOTIFY; + vhost_enable_notify_split(dev, vq, enable); + return 0; } @@ -713,16 +927,9 @@ int rte_vhost_get_log_base(int vid, uint64_t *log_base, { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL || log_base == NULL || log_size == NULL) return -1; - if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { - RTE_LOG(ERR, VHOST_DATA, - "(%d) %s: built-in vhost net backend is disabled.\n", - dev->vid, __func__); - return -1; - } - *log_base = dev->log_base; *log_size = dev->log_size; @@ -734,16 +941,9 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id, { struct virtio_net *dev = get_device(vid); - if (!dev) + if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) return -1; - if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { - RTE_LOG(ERR, VHOST_DATA, - "(%d) %s: built-in vhost net backend is disabled.\n", - dev->vid, __func__); - return -1; - } - *last_avail_idx = dev->virtqueue[queue_id]->last_avail_idx; *last_used_idx = dev->virtqueue[queue_id]->last_used_idx; @@ -758,15 +958,21 @@ int rte_vhost_set_vring_base(int vid, uint16_t queue_id, if (!dev) return -1; - if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { - RTE_LOG(ERR, VHOST_DATA, - "(%d) %s: built-in vhost net backend is disabled.\n", - dev->vid, __func__); - return -1; - } - dev->virtqueue[queue_id]->last_avail_idx = last_avail_idx; dev->virtqueue[queue_id]->last_used_idx = last_used_idx; return 0; } + +int rte_vhost_extern_callback_register(int vid, + struct rte_vhost_user_extern_ops const * const ops, void *ctx) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL || ops == NULL) + return -1; + + dev->extern_ops = *ops; + dev->extern_data = ctx; + return 0; +}