X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost.h;h=cfea138d111e445f5b01f81c31043782d05fccad;hb=c49197ff29fbd67e19a927683b7100dba602718d;hp=d5bab4803a57038296c3e3f994681890b0c8823c;hpb=02e3b285d447f0745b7e09d1ce9717aad55633e5;p=dpdk.git diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index d5bab4803a..cfea138d11 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -18,6 +18,7 @@ #include #include #include +#include #include "rte_vhost.h" #include "rte_vdpa.h" @@ -285,56 +286,6 @@ struct guest_page { uint64_t size; }; -/* The possible results of a message handling function */ -enum vh_result { - /* Message handling failed */ - VH_RESULT_ERR = -1, - /* Message handling successful */ - VH_RESULT_OK = 0, - /* Message handling successful and reply prepared */ - VH_RESULT_REPLY = 1, -}; - -/** - * function prototype for the vhost backend to handler specific vhost user - * messages prior to the master message handling - * - * @param vid - * vhost device id - * @param msg - * Message pointer. - * @param skip_master - * If the handler requires skipping the master message handling, this variable - * shall be written 1, otherwise 0. - * @return - * VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply, - * VH_RESULT_ERR on failure - */ -typedef enum vh_result (*vhost_msg_pre_handle)(int vid, void *msg, - uint32_t *skip_master); - -/** - * function prototype for the vhost backend to handler specific vhost user - * messages after the master message handling is done - * - * @param vid - * vhost device id - * @param msg - * Message pointer. - * @return - * VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply, - * VH_RESULT_ERR on failure - */ -typedef enum vh_result (*vhost_msg_post_handle)(int vid, void *msg); - -/** - * pre and post vhost user message handlers - */ -struct vhost_user_extern_ops { - vhost_msg_pre_handle pre_msg_handle; - vhost_msg_post_handle post_msg_handle; -}; - /** * Device structure contains all configuration information relating * to the device. @@ -357,7 +308,7 @@ struct virtio_net { uint64_t log_size; uint64_t log_base; uint64_t log_addr; - struct ether_addr mac; + struct rte_ether_addr mac; uint16_t mtu; struct vhost_device_ops const *notify_ops; @@ -378,10 +329,10 @@ struct virtio_net { */ int vdpa_dev_id; - /* private data for virtio device */ + /* context data for the external message handlers */ void *extern_data; /* pre and post vhost user message handlers for the device */ - struct vhost_user_extern_ops extern_ops; + struct rte_vhost_user_extern_ops extern_ops; } __rte_cache_aligned; static __rte_always_inline bool @@ -393,161 +344,84 @@ vq_is_packed(struct virtio_net *dev) static inline bool desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) { - uint16_t flags = *((volatile uint16_t *) &desc->flags); + uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE); return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) && wrap_counter != !!(flags & VRING_DESC_F_USED); } -#define VHOST_LOG_PAGE 4096 +void __vhost_log_cache_write(struct virtio_net *dev, + struct vhost_virtqueue *vq, + uint64_t addr, uint64_t len); +void __vhost_log_cache_write_iova(struct virtio_net *dev, + struct vhost_virtqueue *vq, + uint64_t iova, uint64_t len); +void __vhost_log_cache_sync(struct virtio_net *dev, + struct vhost_virtqueue *vq); +void __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len); +void __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t len); -/* - * Atomically set a bit in memory. - */ static __rte_always_inline void -vhost_set_bit(unsigned int nr, volatile uint8_t *addr) +vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) { -#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) - /* - * __sync_ built-ins are deprecated, but __atomic_ ones - * are sub-optimized in older GCC versions. - */ - __sync_fetch_and_or_1(addr, (1U << nr)); -#else - __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); -#endif + if (unlikely(dev->features & (1ULL << VHOST_F_LOG_ALL))) + __vhost_log_write(dev, addr, len); } static __rte_always_inline void -vhost_log_page(uint8_t *log_base, uint64_t page) +vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) { - vhost_set_bit(page % 8, &log_base[page / 8]); + if (unlikely(dev->features & (1ULL << VHOST_F_LOG_ALL))) + __vhost_log_cache_sync(dev, vq); } static __rte_always_inline void -vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) +vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t addr, uint64_t len) { - uint64_t page; - - if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || - !dev->log_base || !len)) - return; - - if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) - return; - - /* To make sure guest memory updates are committed before logging */ - rte_smp_wmb(); - - page = addr / VHOST_LOG_PAGE; - while (page * VHOST_LOG_PAGE < addr + len) { - vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); - page += 1; - } + if (unlikely(dev->features & (1ULL << VHOST_F_LOG_ALL))) + __vhost_log_cache_write(dev, vq, addr, len); } static __rte_always_inline void -vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) +vhost_log_cache_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t offset, uint64_t len) { - unsigned long *log_base; - int i; - - if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || - !dev->log_base)) - return; - - log_base = (unsigned long *)(uintptr_t)dev->log_base; - - /* - * It is expected a write memory barrier has been issued - * before this function is called. - */ - - for (i = 0; i < vq->log_cache_nb_elem; i++) { - struct log_cache_entry *elem = vq->log_cache + i; - -#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) - /* - * '__sync' builtins are deprecated, but '__atomic' ones - * are sub-optimized in older GCC versions. - */ - __sync_fetch_and_or(log_base + elem->offset, elem->val); -#else - __atomic_fetch_or(log_base + elem->offset, elem->val, - __ATOMIC_RELAXED); -#endif - } - - rte_smp_wmb(); - - vq->log_cache_nb_elem = 0; + vhost_log_cache_write(dev, vq, vq->log_guest_addr + offset, len); } static __rte_always_inline void -vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t page) +vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t offset, uint64_t len) { - uint32_t bit_nr = page % (sizeof(unsigned long) << 3); - uint32_t offset = page / (sizeof(unsigned long) << 3); - int i; - - for (i = 0; i < vq->log_cache_nb_elem; i++) { - struct log_cache_entry *elem = vq->log_cache + i; - - if (elem->offset == offset) { - elem->val |= (1UL << bit_nr); - return; - } - } - - if (unlikely(i >= VHOST_LOG_CACHE_NR)) { - /* - * No more room for a new log cache entry, - * so write the dirty log map directly. - */ - rte_smp_wmb(); - vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); - - return; - } - - vq->log_cache[i].offset = offset; - vq->log_cache[i].val = (1UL << bit_nr); - vq->log_cache_nb_elem++; + vhost_log_write(dev, vq->log_guest_addr + offset, len); } static __rte_always_inline void -vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t addr, uint64_t len) +vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t len) { - uint64_t page; - - if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || - !dev->log_base || !len)) + if (likely(!(dev->features & (1ULL << VHOST_F_LOG_ALL)))) return; - if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) - return; - - page = addr / VHOST_LOG_PAGE; - while (page * VHOST_LOG_PAGE < addr + len) { - vhost_log_cache_page(dev, vq, page); - page += 1; - } + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + __vhost_log_cache_write_iova(dev, vq, iova, len); + else + __vhost_log_cache_write(dev, vq, iova, len); } static __rte_always_inline void -vhost_log_cache_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t offset, uint64_t len) +vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t len) { - vhost_log_cache_write(dev, vq, vq->log_guest_addr + offset, len); -} + if (likely(!(dev->features & (1ULL << VHOST_F_LOG_ALL)))) + return; -static __rte_always_inline void -vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t offset, uint64_t len) -{ - vhost_log_write(dev, vq->log_guest_addr + offset, len); + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + __vhost_log_write_iova(dev, vq, iova, len); + else + __vhost_log_write(dev, iova, len); } /* Macros for printing using RTE_LOG */ @@ -604,6 +478,26 @@ gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size) return 0; } +static __rte_always_inline uint64_t +hva_to_gpa(struct virtio_net *dev, uint64_t vva, uint64_t len) +{ + struct rte_vhost_mem_region *r; + uint32_t i; + + if (unlikely(!dev || !dev->mem)) + return 0; + + for (i = 0; i < dev->mem->nregions; i++) { + r = &dev->mem->regions[i]; + + if (vva >= r->host_user_addr && + vva + len < r->host_user_addr + r->size) { + return r->guest_phys_addr + vva - r->host_user_addr; + } + } + return 0; +} + static __rte_always_inline struct virtio_net * get_device(int vid) { @@ -645,6 +539,9 @@ void vhost_backend_cleanup(struct virtio_net *dev); uint64_t __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, uint64_t iova, uint64_t *len, uint8_t perm); +void *vhost_alloc_copy_ind_table(struct virtio_net *dev, + struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len); int vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq); void vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq); @@ -685,21 +582,31 @@ vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq) if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { uint16_t old = vq->signalled_used; uint16_t new = vq->last_used_idx; + bool signalled_used_valid = vq->signalled_used_valid; + + vq->signalled_used = new; + vq->signalled_used_valid = true; VHOST_LOG_DEBUG(VHOST_DATA, "%s: used_event_idx=%d, old=%d, new=%d\n", __func__, vhost_used_event(vq), old, new); - if (vhost_need_event(vhost_used_event(vq), new, old) - && (vq->callfd >= 0)) { - vq->signalled_used = vq->last_used_idx; + + if ((vhost_need_event(vhost_used_event(vq), new, old) && + (vq->callfd >= 0)) || + unlikely(!signalled_used_valid)) { eventfd_write(vq->callfd, (eventfd_t) 1); + if (dev->notify_ops->guest_notified) + dev->notify_ops->guest_notified(dev->vid); } } else { /* Kick the guest if necessary. */ if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) - && (vq->callfd >= 0)) + && (vq->callfd >= 0)) { eventfd_write(vq->callfd, (eventfd_t)1); + if (dev->notify_ops->guest_notified) + dev->notify_ops->guest_notified(dev->vid); + } } } @@ -750,8 +657,51 @@ vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) if (vhost_need_event(off, new, old)) kick = true; kick: - if (kick) + if (kick) { eventfd_write(vq->callfd, (eventfd_t)1); + if (dev->notify_ops->guest_notified) + dev->notify_ops->guest_notified(dev->vid); + } +} + +static __rte_always_inline void +free_ind_table(void *idesc) +{ + rte_free(idesc); +} + +static __rte_always_inline void +restore_mbuf(struct rte_mbuf *m) +{ + uint32_t mbuf_size, priv_size; + + while (m) { + priv_size = rte_pktmbuf_priv_size(m->pool); + mbuf_size = sizeof(struct rte_mbuf) + priv_size; + /* start of buffer is after mbuf structure and priv data */ + + m->buf_addr = (char *)m + mbuf_size; + m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; + m = m->next; + } +} + +static __rte_always_inline bool +mbuf_is_consumed(struct rte_mbuf *m) +{ + while (m) { + if (rte_mbuf_refcnt_read(m) > 1) + return false; + m = m->next; + } + + return true; +} + +static __rte_always_inline void +put_zmbuf(struct zcopy_mbuf *zmbuf) +{ + zmbuf->in_use = 0; } #endif /* _VHOST_NET_CDEV_H_ */