X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=lib%2Flibrte_vhost%2Fvhost.h;h=3b3265c4b63e98232f3b9658a24a4c13fce56826;hb=2a27e0a16d29fbf9c87dba412185684e2671d9ef;hp=b486682c529fe462e2c26674da0eea446d3fe1af;hpb=2d1541e2b6b3b24098d72d5bdecfeed92d83f13a;p=dpdk.git diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index b486682c52..3b3265c4b6 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -18,6 +18,7 @@ #include #include #include +#include #include "rte_vhost.h" #include "rte_vdpa.h" @@ -56,6 +57,7 @@ struct buf_vector { struct zcopy_mbuf { struct rte_mbuf *mbuf; uint32_t desc_idx; + uint16_t desc_count; uint16_t in_use; TAILQ_ENTRY(zcopy_mbuf) next; @@ -80,6 +82,12 @@ struct log_cache_entry { unsigned long val; }; +struct vring_used_elem_packed { + uint16_t id; + uint32_t len; + uint32_t count; +}; + /** * Structure contains variables relevant to RX/TX virtqueues. */ @@ -88,14 +96,21 @@ struct vhost_virtqueue { struct vring_desc *desc; struct vring_packed_desc *desc_packed; }; - struct vring_avail *avail; - struct vring_used *used; + union { + struct vring_avail *avail; + struct vring_packed_desc_event *driver_event; + }; + union { + struct vring_used *used; + struct vring_packed_desc_event *device_event; + }; uint32_t size; uint16_t last_avail_idx; uint16_t last_used_idx; /* Last used index we notify to front end. */ uint16_t signalled_used; + bool signalled_used_valid; #define VIRTIO_INVALID_EVENTFD (-1) #define VIRTIO_UNINITIALIZED_EVENTFD (-2) @@ -119,7 +134,10 @@ struct vhost_virtqueue { struct zcopy_mbuf *zmbufs; struct zcopy_mbuf_list zmbuf_list; - struct vring_used_elem *shadow_used_ring; + union { + struct vring_used_elem *shadow_used_split; + struct vring_used_elem_packed *shadow_used_packed; + }; uint16_t shadow_used_idx; struct vhost_vring_addr ring_addrs; @@ -202,21 +220,30 @@ struct vhost_msg { #define VIRTIO_F_RING_PACKED 34 -#define VRING_DESC_F_NEXT 1 -#define VRING_DESC_F_WRITE 2 -#define VRING_DESC_F_INDIRECT 4 - -#define VRING_DESC_F_AVAIL (1ULL << 7) -#define VRING_DESC_F_USED (1ULL << 15) - struct vring_packed_desc { uint64_t addr; uint32_t len; uint16_t id; uint16_t flags; }; + +struct vring_packed_desc_event { + uint16_t off_wrap; + uint16_t flags; +}; #endif +/* + * Declare below packed ring defines unconditionally + * as Kernel header might use different names. + */ +#define VRING_DESC_F_AVAIL (1ULL << 7) +#define VRING_DESC_F_USED (1ULL << 15) + +#define VRING_EVENT_F_ENABLE 0x0 +#define VRING_EVENT_F_DISABLE 0x1 +#define VRING_EVENT_F_DESC 0x2 + /* * Available and used descs are in same order */ @@ -249,7 +276,8 @@ struct vring_packed_desc { (1ULL << VIRTIO_RING_F_EVENT_IDX) | \ (1ULL << VIRTIO_NET_F_MTU) | \ (1ULL << VIRTIO_F_IN_ORDER) | \ - (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + (1ULL << VIRTIO_F_IOMMU_PLATFORM) | \ + (1ULL << VIRTIO_F_RING_PACKED)) struct guest_page { @@ -258,6 +286,16 @@ struct guest_page { uint64_t size; }; +/* The possible results of a message handling function */ +enum vh_result { + /* Message handling failed */ + VH_RESULT_ERR = -1, + /* Message handling successful */ + VH_RESULT_OK = 0, + /* Message handling successful and reply prepared */ + VH_RESULT_REPLY = 1, +}; + /** * function prototype for the vhost backend to handler specific vhost user * messages prior to the master message handling @@ -266,17 +304,15 @@ struct guest_page { * vhost device id * @param msg * Message pointer. - * @param require_reply - * If the handler requires sending a reply, this varaible shall be written 1, - * otherwise 0. * @param skip_master * If the handler requires skipping the master message handling, this variable * shall be written 1, otherwise 0. * @return - * 0 on success, -1 on failure + * VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply, + * VH_RESULT_ERR on failure */ -typedef int (*vhost_msg_pre_handle)(int vid, void *msg, - uint32_t *require_reply, uint32_t *skip_master); +typedef enum vh_result (*vhost_msg_pre_handle)(int vid, void *msg, + uint32_t *skip_master); /** * function prototype for the vhost backend to handler specific vhost user @@ -286,14 +322,11 @@ typedef int (*vhost_msg_pre_handle)(int vid, void *msg, * vhost device id * @param msg * Message pointer. - * @param require_reply - * If the handler requires sending a reply, this varaible shall be written 1, - * otherwise 0. * @return - * 0 on success, -1 on failure + * VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply, + * VH_RESULT_ERR on failure */ -typedef int (*vhost_msg_post_handle)(int vid, void *msg, - uint32_t *require_reply); +typedef enum vh_result (*vhost_msg_post_handle)(int vid, void *msg); /** * pre and post vhost user message handlers @@ -337,6 +370,9 @@ struct virtio_net { int slave_req_fd; rte_spinlock_t slave_req_lock; + int postcopy_ufd; + int postcopy_listening; + /* * Device id to identify a specific backend device. * It's set to -1 for the default software implementation. @@ -358,8 +394,10 @@ vq_is_packed(struct virtio_net *dev) static inline bool desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) { - return wrap_counter == !!(desc->flags & VRING_DESC_F_AVAIL) && - wrap_counter != !!(desc->flags & VRING_DESC_F_USED); + uint16_t flags = *((volatile uint16_t *) &desc->flags); + + return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) && + wrap_counter != !!(flags & VRING_DESC_F_USED); } #define VHOST_LOG_PAGE 4096 @@ -587,12 +625,11 @@ void vhost_destroy_device(int); void vhost_destroy_device_notify(struct virtio_net *dev); void cleanup_vq(struct vhost_virtqueue *vq, int destroy); -void free_vq(struct vhost_virtqueue *vq); +void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq); int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx); void vhost_attach_vdpa_device(int vid, int did); -void vhost_detach_vdpa_device(int vid); void vhost_set_ifname(int, const char *if_name, unsigned int if_len); void vhost_enable_dequeue_zero_copy(int vid); @@ -622,6 +659,8 @@ vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, return __vhost_iova_to_vva(dev, vq, iova, len, perm); } +#define vhost_avail_event(vr) \ + (*(volatile uint16_t*)&(vr)->used->ring[(vr)->size]) #define vhost_used_event(vr) \ (*(volatile uint16_t*)&(vr)->avail->ring[(vr)->size]) @@ -638,7 +677,7 @@ vhost_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old) } static __rte_always_inline void -vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq) +vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq) { /* Flush used->idx update before we read avail->flags. */ rte_smp_mb(); @@ -665,4 +704,94 @@ vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq) } } +static __rte_always_inline void +vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + uint16_t old, new, off, off_wrap; + bool signalled_used_valid, kick = false; + + /* Flush used desc update. */ + rte_smp_mb(); + + if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { + if (vq->driver_event->flags != + VRING_EVENT_F_DISABLE) + kick = true; + goto kick; + } + + old = vq->signalled_used; + new = vq->last_used_idx; + vq->signalled_used = new; + signalled_used_valid = vq->signalled_used_valid; + vq->signalled_used_valid = true; + + if (vq->driver_event->flags != VRING_EVENT_F_DESC) { + if (vq->driver_event->flags != VRING_EVENT_F_DISABLE) + kick = true; + goto kick; + } + + if (unlikely(!signalled_used_valid)) { + kick = true; + goto kick; + } + + rte_smp_rmb(); + + off_wrap = vq->driver_event->off_wrap; + off = off_wrap & ~(1 << 15); + + if (new <= old) + old -= vq->size; + + if (vq->used_wrap_counter != off_wrap >> 15) + off -= vq->size; + + if (vhost_need_event(off, new, old)) + kick = true; +kick: + if (kick) + eventfd_write(vq->callfd, (eventfd_t)1); +} + +static __rte_always_inline void * +alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len) +{ + void *idesc; + uint64_t src, dst; + uint64_t len, remain = desc_len; + + idesc = rte_malloc(__func__, desc_len, 0); + if (unlikely(!idesc)) + return 0; + + dst = (uint64_t)(uintptr_t)idesc; + + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, desc_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + rte_free(idesc); + return 0; + } + + rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + desc_addr += len; + } + + return idesc; +} + +static __rte_always_inline void +free_ind_table(void *idesc) +{ + rte_free(idesc); +} + #endif /* _VHOST_NET_CDEV_H_ */