X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost.h;h=d49c3b8a98cacb4ef3436a6374fed4e6cb3edfec;hb=6d13ea8e8e49ab957deae2bba5ecf4a4bfe747d1;hp=d362823a946f451d0546cc70ea061e497a42a186;hpb=d3211c98c45672f700e270d98b1dbca62b634c72;p=dpdk.git diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index d362823a94..d49c3b8a98 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -18,6 +18,7 @@ #include #include #include +#include #include "rte_vhost.h" #include "rte_vdpa.h" @@ -56,6 +57,7 @@ struct buf_vector { struct zcopy_mbuf { struct rte_mbuf *mbuf; uint32_t desc_idx; + uint16_t desc_count; uint16_t in_use; TAILQ_ENTRY(zcopy_mbuf) next; @@ -80,19 +82,35 @@ struct log_cache_entry { unsigned long val; }; +struct vring_used_elem_packed { + uint16_t id; + uint32_t len; + uint32_t count; +}; + /** * Structure contains variables relevant to RX/TX virtqueues. */ struct vhost_virtqueue { - struct vring_desc *desc; - struct vring_avail *avail; - struct vring_used *used; + union { + struct vring_desc *desc; + struct vring_packed_desc *desc_packed; + }; + union { + struct vring_avail *avail; + struct vring_packed_desc_event *driver_event; + }; + union { + struct vring_used *used; + struct vring_packed_desc_event *device_event; + }; uint32_t size; uint16_t last_avail_idx; uint16_t last_used_idx; /* Last used index we notify to front end. */ uint16_t signalled_used; + bool signalled_used_valid; #define VIRTIO_INVALID_EVENTFD (-1) #define VIRTIO_UNINITIALIZED_EVENTFD (-2) @@ -116,12 +134,17 @@ struct vhost_virtqueue { struct zcopy_mbuf *zmbufs; struct zcopy_mbuf_list zmbuf_list; - struct vring_used_elem *shadow_used_ring; + union { + struct vring_used_elem *shadow_used_split; + struct vring_used_elem_packed *shadow_used_packed; + }; uint16_t shadow_used_idx; struct vhost_vring_addr ring_addrs; struct batch_copy_elem *batch_copy_elems; uint16_t batch_copy_nb_elems; + bool used_wrap_counter; + bool avail_wrap_counter; struct log_cache_entry log_cache[VHOST_LOG_CACHE_NR]; uint16_t log_cache_nb_elem; @@ -197,21 +220,30 @@ struct vhost_msg { #define VIRTIO_F_RING_PACKED 34 -#define VRING_DESC_F_NEXT 1 -#define VRING_DESC_F_WRITE 2 -#define VRING_DESC_F_INDIRECT 4 - -#define VRING_DESC_F_AVAIL (1ULL << 7) -#define VRING_DESC_F_USED (1ULL << 15) - struct vring_packed_desc { uint64_t addr; uint32_t len; uint16_t id; uint16_t flags; }; + +struct vring_packed_desc_event { + uint16_t off_wrap; + uint16_t flags; +}; #endif +/* + * Declare below packed ring defines unconditionally + * as Kernel header might use different names. + */ +#define VRING_DESC_F_AVAIL (1ULL << 7) +#define VRING_DESC_F_USED (1ULL << 15) + +#define VRING_EVENT_F_ENABLE 0x0 +#define VRING_EVENT_F_DISABLE 0x1 +#define VRING_EVENT_F_DESC 0x2 + /* * Available and used descs are in same order */ @@ -244,7 +276,8 @@ struct vring_packed_desc { (1ULL << VIRTIO_RING_F_EVENT_IDX) | \ (1ULL << VIRTIO_NET_F_MTU) | \ (1ULL << VIRTIO_F_IN_ORDER) | \ - (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + (1ULL << VIRTIO_F_IOMMU_PLATFORM) | \ + (1ULL << VIRTIO_F_RING_PACKED)) struct guest_page { @@ -253,51 +286,6 @@ struct guest_page { uint64_t size; }; -/** - * function prototype for the vhost backend to handler specific vhost user - * messages prior to the master message handling - * - * @param vid - * vhost device id - * @param msg - * Message pointer. - * @param require_reply - * If the handler requires sending a reply, this varaible shall be written 1, - * otherwise 0. - * @param skip_master - * If the handler requires skipping the master message handling, this variable - * shall be written 1, otherwise 0. - * @return - * 0 on success, -1 on failure - */ -typedef int (*vhost_msg_pre_handle)(int vid, void *msg, - uint32_t *require_reply, uint32_t *skip_master); - -/** - * function prototype for the vhost backend to handler specific vhost user - * messages after the master message handling is done - * - * @param vid - * vhost device id - * @param msg - * Message pointer. - * @param require_reply - * If the handler requires sending a reply, this varaible shall be written 1, - * otherwise 0. - * @return - * 0 on success, -1 on failure - */ -typedef int (*vhost_msg_post_handle)(int vid, void *msg, - uint32_t *require_reply); - -/** - * pre and post vhost user message handlers - */ -struct vhost_user_extern_ops { - vhost_msg_pre_handle pre_msg_handle; - vhost_msg_post_handle post_msg_handle; -}; - /** * Device structure contains all configuration information relating * to the device. @@ -320,7 +308,7 @@ struct virtio_net { uint64_t log_size; uint64_t log_base; uint64_t log_addr; - struct ether_addr mac; + struct rte_ether_addr mac; uint16_t mtu; struct vhost_device_ops const *notify_ops; @@ -332,16 +320,19 @@ struct virtio_net { int slave_req_fd; rte_spinlock_t slave_req_lock; + int postcopy_ufd; + int postcopy_listening; + /* * Device id to identify a specific backend device. * It's set to -1 for the default software implementation. */ int vdpa_dev_id; - /* private data for virtio device */ + /* context data for the external message handlers */ void *extern_data; /* pre and post vhost user message handlers for the device */ - struct vhost_user_extern_ops extern_ops; + struct rte_vhost_user_extern_ops extern_ops; } __rte_cache_aligned; static __rte_always_inline bool @@ -353,8 +344,10 @@ vq_is_packed(struct virtio_net *dev) static inline bool desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) { - return wrap_counter == !!(desc->flags & VRING_DESC_F_AVAIL) && - wrap_counter != !!(desc->flags & VRING_DESC_F_USED); + uint16_t flags = *((volatile uint16_t *) &desc->flags); + + return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) && + wrap_counter != !!(flags & VRING_DESC_F_USED); } #define VHOST_LOG_PAGE 4096 @@ -414,12 +407,9 @@ vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) !dev->log_base)) return; - log_base = (unsigned long *)(uintptr_t)dev->log_base; + rte_smp_wmb(); - /* - * It is expected a write memory barrier has been issued - * before this function is called. - */ + log_base = (unsigned long *)(uintptr_t)dev->log_base; for (i = 0; i < vq->log_cache_nb_elem; i++) { struct log_cache_entry *elem = vq->log_cache + i; @@ -582,12 +572,11 @@ void vhost_destroy_device(int); void vhost_destroy_device_notify(struct virtio_net *dev); void cleanup_vq(struct vhost_virtqueue *vq, int destroy); -void free_vq(struct vhost_virtqueue *vq); +void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq); int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx); void vhost_attach_vdpa_device(int vid, int did); -void vhost_detach_vdpa_device(int vid); void vhost_set_ifname(int, const char *if_name, unsigned int if_len); void vhost_enable_dequeue_zero_copy(int vid); @@ -617,6 +606,8 @@ vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, return __vhost_iova_to_vva(dev, vq, iova, len, perm); } +#define vhost_avail_event(vr) \ + (*(volatile uint16_t*)&(vr)->used->ring[(vr)->size]) #define vhost_used_event(vr) \ (*(volatile uint16_t*)&(vr)->avail->ring[(vr)->size]) @@ -633,7 +624,7 @@ vhost_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old) } static __rte_always_inline void -vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq) +vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq) { /* Flush used->idx update before we read avail->flags. */ rte_smp_mb(); @@ -642,16 +633,20 @@ vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq) if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { uint16_t old = vq->signalled_used; uint16_t new = vq->last_used_idx; + bool signalled_used_valid = vq->signalled_used_valid; + + vq->signalled_used = new; + vq->signalled_used_valid = true; VHOST_LOG_DEBUG(VHOST_DATA, "%s: used_event_idx=%d, old=%d, new=%d\n", __func__, vhost_used_event(vq), old, new); - if (vhost_need_event(vhost_used_event(vq), new, old) - && (vq->callfd >= 0)) { - vq->signalled_used = vq->last_used_idx; + + if ((vhost_need_event(vhost_used_event(vq), new, old) && + (vq->callfd >= 0)) || + unlikely(!signalled_used_valid)) eventfd_write(vq->callfd, (eventfd_t) 1); - } } else { /* Kick the guest if necessary. */ if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) @@ -660,4 +655,128 @@ vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq) } } +static __rte_always_inline void +vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + uint16_t old, new, off, off_wrap; + bool signalled_used_valid, kick = false; + + /* Flush used desc update. */ + rte_smp_mb(); + + if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { + if (vq->driver_event->flags != + VRING_EVENT_F_DISABLE) + kick = true; + goto kick; + } + + old = vq->signalled_used; + new = vq->last_used_idx; + vq->signalled_used = new; + signalled_used_valid = vq->signalled_used_valid; + vq->signalled_used_valid = true; + + if (vq->driver_event->flags != VRING_EVENT_F_DESC) { + if (vq->driver_event->flags != VRING_EVENT_F_DISABLE) + kick = true; + goto kick; + } + + if (unlikely(!signalled_used_valid)) { + kick = true; + goto kick; + } + + rte_smp_rmb(); + + off_wrap = vq->driver_event->off_wrap; + off = off_wrap & ~(1 << 15); + + if (new <= old) + old -= vq->size; + + if (vq->used_wrap_counter != off_wrap >> 15) + off -= vq->size; + + if (vhost_need_event(off, new, old)) + kick = true; +kick: + if (kick) + eventfd_write(vq->callfd, (eventfd_t)1); +} + +static __rte_always_inline void * +alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len) +{ + void *idesc; + uint64_t src, dst; + uint64_t len, remain = desc_len; + + idesc = rte_malloc(__func__, desc_len, 0); + if (unlikely(!idesc)) + return 0; + + dst = (uint64_t)(uintptr_t)idesc; + + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, desc_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + rte_free(idesc); + return 0; + } + + rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + desc_addr += len; + } + + return idesc; +} + +static __rte_always_inline void +free_ind_table(void *idesc) +{ + rte_free(idesc); +} + +static __rte_always_inline void +restore_mbuf(struct rte_mbuf *m) +{ + uint32_t mbuf_size, priv_size; + + while (m) { + priv_size = rte_pktmbuf_priv_size(m->pool); + mbuf_size = sizeof(struct rte_mbuf) + priv_size; + /* start of buffer is after mbuf structure and priv data */ + + m->buf_addr = (char *)m + mbuf_size; + m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; + m = m->next; + } +} + +static __rte_always_inline bool +mbuf_is_consumed(struct rte_mbuf *m) +{ + while (m) { + if (rte_mbuf_refcnt_read(m) > 1) + return false; + m = m->next; + } + + return true; +} + +static __rte_always_inline void +put_zmbuf(struct zcopy_mbuf *zmbuf) +{ + zmbuf->in_use = 0; +} + #endif /* _VHOST_NET_CDEV_H_ */