net/virtio: improve perf via one-way barriers on used flag

[dpdk.git] / lib / librte_vhost / vhost_user.c

diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 0b72648..ce4e9fb 100644 (file)
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -410,6 +410,9 @@ numa_realloc(struct virtio_net *dev, int index)
        struct batch_copy_elem *new_batch_copy_elems;
        int ret;
 
+       if (dev->flags & VIRTIO_DEV_RUNNING)
+               return dev;
+
        old_dev = dev;
        vq = old_vq = dev->virtqueue[index];
 
@@ -558,11 +561,13 @@ ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 {
        if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
                uint64_t vva;
+               uint64_t req_size = *size;
 
                vva = vhost_user_iotlb_cache_find(vq, ra,
                                        size, VHOST_ACCESS_RW);
-               if (!vva)
-                       vhost_user_iotlb_miss(dev, ra, VHOST_ACCESS_RW);
+               if (req_size != *size)
+                       vhost_user_iotlb_miss(dev, (ra + *size),
+                                             VHOST_ACCESS_RW);
 
                return vva;
        }
@@ -619,6 +624,7 @@ translate_ring_addresses(struct virtio_net *dev, int vq_index)
                        return dev;
                }
 
+               vq->access_ok = 1;
                return dev;
        }
 
@@ -677,6 +683,7 @@ translate_ring_addresses(struct virtio_net *dev, int vq_index)
        }
 
        vq->log_guest_addr = addr->log_guest_addr;
+       vq->access_ok = 1;
 
        VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
                        dev->vid, vq->desc);
@@ -701,6 +708,7 @@ vhost_user_set_vring_addr(struct virtio_net **pdev, struct VhostUserMsg *msg,
        struct virtio_net *dev = *pdev;
        struct vhost_virtqueue *vq;
        struct vhost_vring_addr *addr = &msg->payload.addr;
+       bool access_ok;
 
        if (dev->mem == NULL)
                return RTE_VHOST_MSG_RESULT_ERR;
@@ -708,6 +716,8 @@ vhost_user_set_vring_addr(struct virtio_net **pdev, struct VhostUserMsg *msg,
        /* addr->index refers to the queue index. The txq 1, rxq is 0. */
        vq = dev->virtqueue[msg->payload.addr.index];
 
+       access_ok = vq->access_ok;
+
        /*
         * Rings addresses should not be interpreted as long as the ring is not
         * started and enabled
@@ -716,8 +726,9 @@ vhost_user_set_vring_addr(struct virtio_net **pdev, struct VhostUserMsg *msg,
 
        vring_invalidate(dev, vq);
 
-       if (vq->enabled && (dev->features &
-                               (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) {
+       if ((vq->enabled && (dev->features &
+                               (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) ||
+                       access_ok) {
                dev = translate_ring_addresses(dev, msg->payload.addr.index);
                if (!dev)
                        return RTE_VHOST_MSG_RESULT_ERR;
@@ -1322,6 +1333,8 @@ vhost_user_get_vring_base(struct virtio_net **pdev,
        msg->size = sizeof(msg->payload.state);
        msg->fd_num = 0;
 
+       vring_invalidate(dev, vq);
+
        return RTE_VHOST_MSG_RESULT_REPLY;
 }
 
@@ -1564,60 +1577,75 @@ vhost_user_set_req_fd(struct virtio_net **pdev, struct VhostUserMsg *msg,
                return RTE_VHOST_MSG_RESULT_ERR;
        }
 
+       if (dev->slave_req_fd >= 0)
+               close(dev->slave_req_fd);
+
        dev->slave_req_fd = fd;
 
        return RTE_VHOST_MSG_RESULT_OK;
 }
 
 static int
-is_vring_iotlb_update(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
+is_vring_iotlb_split(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
 {
        struct vhost_vring_addr *ra;
-       uint64_t start, end;
+       uint64_t start, end, len;
 
        start = imsg->iova;
        end = start + imsg->size;
 
        ra = &vq->ring_addrs;
-       if (ra->desc_user_addr >= start && ra->desc_user_addr < end)
+       len = sizeof(struct vring_desc) * vq->size;
+       if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
                return 1;
-       if (ra->avail_user_addr >= start && ra->avail_user_addr < end)
+
+       len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
+       if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
                return 1;
-       if (ra->used_user_addr >= start && ra->used_user_addr < end)
+
+       len = sizeof(struct vring_used) +
+              sizeof(struct vring_used_elem) * vq->size;
+       if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
                return 1;
 
        return 0;
 }
 
 static int
-is_vring_iotlb_invalidate(struct vhost_virtqueue *vq,
-                               struct vhost_iotlb_msg *imsg)
+is_vring_iotlb_packed(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
 {
-       uint64_t istart, iend, vstart, vend;
+       struct vhost_vring_addr *ra;
+       uint64_t start, end, len;
 
-       istart = imsg->iova;
-       iend = istart + imsg->size - 1;
+       start = imsg->iova;
+       end = start + imsg->size;
 
-       vstart = (uintptr_t)vq->desc;
-       vend = vstart + sizeof(struct vring_desc) * vq->size - 1;
-       if (vstart <= iend && istart <= vend)
+       ra = &vq->ring_addrs;
+       len = sizeof(struct vring_packed_desc) * vq->size;
+       if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
                return 1;
 
-       vstart = (uintptr_t)vq->avail;
-       vend = vstart + sizeof(struct vring_avail);
-       vend += sizeof(uint16_t) * vq->size - 1;
-       if (vstart <= iend && istart <= vend)
+       len = sizeof(struct vring_packed_desc_event);
+       if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
                return 1;
 
-       vstart = (uintptr_t)vq->used;
-       vend = vstart + sizeof(struct vring_used);
-       vend += sizeof(struct vring_used_elem) * vq->size - 1;
-       if (vstart <= iend && istart <= vend)
+       len = sizeof(struct vring_packed_desc_event);
+       if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
                return 1;
 
        return 0;
 }
 
+static int is_vring_iotlb(struct virtio_net *dev,
+                         struct vhost_virtqueue *vq,
+                         struct vhost_iotlb_msg *imsg)
+{
+       if (vq_is_packed(dev))
+               return is_vring_iotlb_packed(vq, imsg);
+       else
+               return is_vring_iotlb_split(vq, imsg);
+}
+
 static int
 vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg,
                        int main_fd __rte_unused)
@@ -1640,7 +1668,7 @@ vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg,
                        vhost_user_iotlb_cache_insert(vq, imsg->iova, vva,
                                        len, imsg->perm);
 
-                       if (is_vring_iotlb_update(vq, imsg))
+                       if (is_vring_iotlb(dev, vq, imsg))
                                *pdev = dev = translate_ring_addresses(dev, i);
                }
                break;
@@ -1651,7 +1679,7 @@ vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg,
                        vhost_user_iotlb_cache_remove(vq, imsg->iova,
                                        imsg->size);
 
-                       if (is_vring_iotlb_invalidate(vq, imsg))
+                       if (is_vring_iotlb(dev, vq, imsg))
                                vring_invalidate(dev, vq);
                }
                break;