mbuf: rename physical address to IOVA

[dpdk.git] / lib / librte_vhost / virtio_net.c

diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 59ff6c8..6fee16e 100644 (file)
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -45,6 +45,7 @@
 #include <rte_sctp.h>
 #include <rte_arp.h>
 
+#include "iotlb.h"
 #include "vhost.h"
 
 #define MAX_PKT_BURST 32
@@ -211,7 +212,8 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
        int error = 0;
 
        desc = &descs[desc_idx];
-       desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+       desc_addr = vhost_iova_to_vva(dev, vq, desc->addr,
+                                       desc->len, VHOST_ACCESS_RW);
        /*
         * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
         * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
@@ -255,7 +257,9 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
                        }
 
                        desc = &descs[desc->next];
-                       desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+                       desc_addr = vhost_iova_to_vva(dev, vq, desc->addr,
+                                                       desc->len,
+                                                       VHOST_ACCESS_RW);
                        if (unlikely(!desc_addr)) {
                                error = -1;
                                goto out;
@@ -325,13 +329,23 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
        if (unlikely(vq->enabled == 0))
                return 0;
 
+       if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+               vhost_user_iotlb_rd_lock(vq);
+
+       if (unlikely(vq->access_ok == 0)) {
+               if (unlikely(vring_translate(dev, vq) < 0)) {
+                       count = 0;
+                       goto out;
+               }
+       }
+
        avail_idx = *((volatile uint16_t *)&vq->avail->idx);
        start_idx = vq->last_used_idx;
        free_entries = avail_idx - start_idx;
        count = RTE_MIN(count, free_entries);
        count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST);
        if (count == 0)
-               return 0;
+               goto out;
 
        LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n",
                dev->vid, start_idx, start_idx + count);
@@ -358,8 +372,10 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
 
                if (vq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) {
                        descs = (struct vring_desc *)(uintptr_t)
-                               rte_vhost_gpa_to_vva(dev->mem,
-                                       vq->desc[desc_idx].addr);
+                               vhost_iova_to_vva(dev,
+                                               vq, vq->desc[desc_idx].addr,
+                                               vq->desc[desc_idx].len,
+                                               VHOST_ACCESS_RO);
                        if (unlikely(!descs)) {
                                count = i;
                                break;
@@ -399,6 +415,10 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
        if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
                        && (vq->callfd >= 0))
                eventfd_write(vq->callfd, (eventfd_t)1);
+out:
+       if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+               vhost_user_iotlb_rd_unlock(vq);
+
        return count;
 }
 
@@ -417,7 +437,9 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
        if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
                descs = (struct vring_desc *)(uintptr_t)
-                       rte_vhost_gpa_to_vva(dev->mem, vq->desc[idx].addr);
+                       vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
+                                               vq->desc[idx].len,
+                                               VHOST_ACCESS_RO);
                if (unlikely(!descs))
                        return -1;
 
@@ -512,7 +534,9 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
                goto out;
        }
 
-       desc_addr = rte_vhost_gpa_to_vva(dev->mem, buf_vec[vec_idx].buf_addr);
+       desc_addr = vhost_iova_to_vva(dev, vq, buf_vec[vec_idx].buf_addr,
+                                               buf_vec[vec_idx].buf_len,
+                                               VHOST_ACCESS_RW);
        if (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) {
                error = -1;
                goto out;
@@ -535,8 +559,11 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
                /* done with current desc buf, get the next one */
                if (desc_avail == 0) {
                        vec_idx++;
-                       desc_addr = rte_vhost_gpa_to_vva(dev->mem,
-                                       buf_vec[vec_idx].buf_addr);
+                       desc_addr =
+                               vhost_iova_to_vva(dev, vq,
+                                       buf_vec[vec_idx].buf_addr,
+                                       buf_vec[vec_idx].buf_len,
+                                       VHOST_ACCESS_RW);
                        if (unlikely(!desc_addr)) {
                                error = -1;
                                goto out;
@@ -627,9 +654,16 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
        if (unlikely(vq->enabled == 0))
                return 0;
 
+       if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+               vhost_user_iotlb_rd_lock(vq);
+
+       if (unlikely(vq->access_ok == 0))
+               if (unlikely(vring_translate(dev, vq) < 0))
+                       goto out;
+
        count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
        if (count == 0)
-               return 0;
+               goto out;
 
        vq->batch_copy_nb_elems = 0;
 
@@ -677,6 +711,10 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
                        eventfd_write(vq->callfd, (eventfd_t)1);
        }
 
+out:
+       if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+               vhost_user_iotlb_rd_unlock(vq);
+
        return pkt_idx;
 }
 
@@ -875,7 +913,10 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
                goto out;
        }
 
-       desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+       desc_addr = vhost_iova_to_vva(dev,
+                                       vq, desc->addr,
+                                       desc->len,
+                                       VHOST_ACCESS_RO);
        if (unlikely(!desc_addr)) {
                error = -1;
                goto out;
@@ -899,7 +940,10 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
                        goto out;
                }
 
-               desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+               desc_addr = vhost_iova_to_vva(dev,
+                                                       vq, desc->addr,
+                                                       desc->len,
+                                                       VHOST_ACCESS_RO);
                if (unlikely(!desc_addr)) {
                        error = -1;
                        goto out;
@@ -934,7 +978,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
                        cur->data_len = cpy_len;
                        cur->data_off = 0;
                        cur->buf_addr = (void *)(uintptr_t)desc_addr;
-                       cur->buf_physaddr = hpa;
+                       cur->buf_iova = hpa;
 
                        /*
                         * In zero copy mode, one mbuf can only reference data
@@ -943,7 +987,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
                        mbuf_avail = cpy_len;
                } else {
                        if (likely(cpy_len > MAX_BATCH_LEN ||
-                                  copy_nb >= vq->size)) {
+                                  copy_nb >= vq->size ||
+                                  (hdr && cur == m))) {
                                rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
                                                                   mbuf_offset),
                                           (void *)((uintptr_t)(desc_addr +
@@ -982,7 +1027,10 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
                                goto out;
                        }
 
-                       desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+                       desc_addr = vhost_iova_to_vva(dev,
+                                                       vq, desc->addr,
+                                                       desc->len,
+                                                       VHOST_ACCESS_RO);
                        if (unlikely(!desc_addr)) {
                                error = -1;
                                goto out;
@@ -1137,6 +1185,13 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
 
        vq->batch_copy_nb_elems = 0;
 
+       if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+               vhost_user_iotlb_rd_lock(vq);
+
+       if (unlikely(vq->access_ok == 0))
+               if (unlikely(vring_translate(dev, vq) < 0))
+                       goto out;
+
        if (unlikely(dev->dequeue_zero_copy)) {
                struct zcopy_mbuf *zmbuf, *next;
                int nr_updated = 0;
@@ -1236,8 +1291,10 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
 
                if (vq->desc[desc_indexes[i]].flags & VRING_DESC_F_INDIRECT) {
                        desc = (struct vring_desc *)(uintptr_t)
-                               rte_vhost_gpa_to_vva(dev->mem,
-                                       vq->desc[desc_indexes[i]].addr);
+                               vhost_iova_to_vva(dev, vq,
+                                               vq->desc[desc_indexes[i]].addr,
+                                               sizeof(*desc),
+                                               VHOST_ACCESS_RO);
                        if (unlikely(!desc))
                                break;
 
@@ -1296,6 +1353,9 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
        }
 
 out:
+       if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+               vhost_user_iotlb_rd_unlock(vq);
+
        if (unlikely(rarp_mbuf != NULL)) {
                /*
                 * Inject it to the head of "pkts" array, so that switch's mac