vhost: fix async copy on multi-page buffers

[dpdk.git] / lib / librte_vhost / vhost.h

diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 8c01cee..632f66d 100644 (file)
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -46,6 +46,8 @@
 
 #define MAX_PKT_BURST 32
 
+#define ASYNC_MAX_POLL_SEG 255
+
 #define VHOST_MAX_ASYNC_IT (MAX_PKT_BURST * 2)
 #define VHOST_MAX_ASYNC_VEC (BUF_VECTOR_MAX * 2)
 
@@ -162,6 +164,9 @@ struct vhost_virtqueue {
        int                     enabled;
        int                     access_ok;
        int                     ready;
+       int                     notif_enable;
+#define VIRTIO_UNINITIALIZED_NOTIF     (-1)
+
        rte_spinlock_t          access_lock;
 
        /* Used to notify the guest (trigger interrupt) */
@@ -225,6 +230,7 @@ struct vhost_virtqueue {
        uint64_t        *async_pending_info;
        uint16_t        async_pkts_idx;
        uint16_t        async_pkts_inflight_n;
+       uint16_t        async_last_seg_n;
 
        /* vq async features */
        bool            async_inorder;
@@ -613,6 +619,56 @@ gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
        return 0;
 }
 
+static __rte_always_inline rte_iova_t
+gpa_to_first_hpa(struct virtio_net *dev, uint64_t gpa,
+       uint64_t gpa_size, uint64_t *hpa_size)
+{
+       uint32_t i;
+       struct guest_page *page;
+       struct guest_page key;
+
+       *hpa_size = gpa_size;
+       if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
+               key.guest_phys_addr = gpa & ~(dev->guest_pages[0].size - 1);
+               page = bsearch(&key, dev->guest_pages, dev->nr_guest_pages,
+                              sizeof(struct guest_page), guest_page_addrcmp);
+               if (page) {
+                       if (gpa + gpa_size <=
+                                       page->guest_phys_addr + page->size) {
+                               return gpa - page->guest_phys_addr +
+                                       page->host_phys_addr;
+                       } else if (gpa < page->guest_phys_addr +
+                                               page->size) {
+                               *hpa_size = page->guest_phys_addr +
+                                       page->size - gpa;
+                               return gpa - page->guest_phys_addr +
+                                       page->host_phys_addr;
+                       }
+               }
+       } else {
+               for (i = 0; i < dev->nr_guest_pages; i++) {
+                       page = &dev->guest_pages[i];
+
+                       if (gpa >= page->guest_phys_addr) {
+                               if (gpa + gpa_size <=
+                                       page->guest_phys_addr + page->size) {
+                                       return gpa - page->guest_phys_addr +
+                                               page->host_phys_addr;
+                               } else if (gpa < page->guest_phys_addr +
+                                                       page->size) {
+                                       *hpa_size = page->guest_phys_addr +
+                                               page->size - gpa;
+                                       return gpa - page->guest_phys_addr +
+                                               page->host_phys_addr;
+                               }
+                       }
+               }
+       }
+
+       *hpa_size = 0;
+       return 0;
+}
+
 static __rte_always_inline uint64_t
 hva_to_gpa(struct virtio_net *dev, uint64_t vva, uint64_t len)
 {
@@ -665,6 +721,8 @@ void vhost_enable_dequeue_zero_copy(int vid);
 void vhost_set_builtin_virtio_net(int vid, bool enable);
 void vhost_enable_extbuf(int vid);
 void vhost_enable_linearbuf(int vid);
+int vhost_enable_guest_notification(struct virtio_net *dev,
+               struct vhost_virtqueue *vq, int enable);
 
 struct vhost_device_ops const *vhost_driver_callback_get(const char *path);