lib/librte_vhost/iotlb.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2017 Red Hat, Inc.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #ifdef RTE_LIBRTE_VHOST_NUMA
  34 #include <numaif.h>
  35 #endif
  36
  37 #include <rte_tailq.h>
  38
  39 #include "iotlb.h"
  40 #include "vhost.h"
  41
  42 struct vhost_iotlb_entry {
  43         TAILQ_ENTRY(vhost_iotlb_entry) next;
  44
  45         uint64_t iova;
  46         uint64_t uaddr;
  47         uint64_t size;
  48         uint8_t perm;
  49 };
  50
  51 #define IOTLB_CACHE_SIZE 1024
  52
  53 static void
  54 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
  55 {
  56         struct vhost_iotlb_entry *node, *temp_node;
  57
  58         rte_rwlock_write_lock(&vq->iotlb_lock);
  59
  60         TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
  61                 TAILQ_REMOVE(&vq->iotlb_list, node, next);
  62                 rte_mempool_put(vq->iotlb_pool, node);
  63         }
  64
  65         vq->iotlb_cache_nr = 0;
  66
  67         rte_rwlock_write_unlock(&vq->iotlb_lock);
  68 }
  69
  70 static void
  71 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
  72 {
  73         struct vhost_iotlb_entry *node, *temp_node;
  74         int entry_idx;
  75
  76         rte_rwlock_write_lock(&vq->iotlb_lock);
  77
  78         entry_idx = rte_rand() % vq->iotlb_cache_nr;
  79
  80         TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
  81                 if (!entry_idx) {
  82                         TAILQ_REMOVE(&vq->iotlb_list, node, next);
  83                         rte_mempool_put(vq->iotlb_pool, node);
  84                         vq->iotlb_cache_nr--;
  85                         break;
  86                 }
  87                 entry_idx--;
  88         }
  89
  90         rte_rwlock_write_unlock(&vq->iotlb_lock);
  91 }
  92
  93 void
  94 vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
  95                                 uint64_t uaddr, uint64_t size, uint8_t perm)
  96 {
  97         struct vhost_iotlb_entry *node, *new_node;
  98         int ret;
  99
 100         ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 101         if (ret) {
 102                 RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, evict one entry\n");
 103                 vhost_user_iotlb_cache_random_evict(vq);
 104                 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 105                 if (ret) {
 106                         RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n");
 107                         return;
 108                 }
 109         }
 110
 111         new_node->iova = iova;
 112         new_node->uaddr = uaddr;
 113         new_node->size = size;
 114         new_node->perm = perm;
 115
 116         rte_rwlock_write_lock(&vq->iotlb_lock);
 117
 118         TAILQ_FOREACH(node, &vq->iotlb_list, next) {
 119                 /*
 120                  * Entries must be invalidated before being updated.
 121                  * So if iova already in list, assume identical.
 122                  */
 123                 if (node->iova == new_node->iova) {
 124                         rte_mempool_put(vq->iotlb_pool, new_node);
 125                         goto unlock;
 126                 } else if (node->iova > new_node->iova) {
 127                         TAILQ_INSERT_BEFORE(node, new_node, next);
 128                         vq->iotlb_cache_nr++;
 129                         goto unlock;
 130                 }
 131         }
 132
 133         TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
 134         vq->iotlb_cache_nr++;
 135
 136 unlock:
 137         rte_rwlock_write_unlock(&vq->iotlb_lock);
 138 }
 139
 140 void
 141 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
 142                                         uint64_t iova, uint64_t size)
 143 {
 144         struct vhost_iotlb_entry *node, *temp_node;
 145
 146         if (unlikely(!size))
 147                 return;
 148
 149         rte_rwlock_write_lock(&vq->iotlb_lock);
 150
 151         TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
 152                 /* Sorted list */
 153                 if (unlikely(iova + size < node->iova))
 154                         break;
 155
 156                 if (iova < node->iova + node->size) {
 157                         TAILQ_REMOVE(&vq->iotlb_list, node, next);
 158                         rte_mempool_put(vq->iotlb_pool, node);
 159                         vq->iotlb_cache_nr--;
 160                 }
 161         }
 162
 163         rte_rwlock_write_unlock(&vq->iotlb_lock);
 164 }
 165
 166 uint64_t
 167 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
 168                                                 uint64_t *size, uint8_t perm)
 169 {
 170         struct vhost_iotlb_entry *node;
 171         uint64_t offset, vva = 0, mapped = 0;
 172
 173         if (unlikely(!*size))
 174                 goto out;
 175
 176         TAILQ_FOREACH(node, &vq->iotlb_list, next) {
 177                 /* List sorted by iova */
 178                 if (unlikely(iova < node->iova))
 179                         break;
 180
 181                 if (iova >= node->iova + node->size)
 182                         continue;
 183
 184                 if (unlikely((perm & node->perm) != perm)) {
 185                         vva = 0;
 186                         break;
 187                 }
 188
 189                 offset = iova - node->iova;
 190                 if (!vva)
 191                         vva = node->uaddr + offset;
 192
 193                 mapped += node->size - offset;
 194                 iova = node->iova + node->size;
 195
 196                 if (mapped >= *size)
 197                         break;
 198         }
 199
 200 out:
 201         /* Only part of the requested chunk is mapped */
 202         if (unlikely(mapped < *size))
 203                 *size = mapped;
 204
 205         return vva;
 206 }
 207
 208 int
 209 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
 210 {
 211         char pool_name[RTE_MEMPOOL_NAMESIZE];
 212         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 213         int ret = -1, socket;
 214
 215         if (vq->iotlb_pool) {
 216                 /*
 217                  * The cache has already been initialized,
 218                  * just drop all entries
 219                  */
 220                 vhost_user_iotlb_cache_remove_all(vq);
 221                 return 0;
 222         }
 223
 224 #ifdef RTE_LIBRTE_VHOST_NUMA
 225         ret = get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR);
 226 #endif
 227         if (ret)
 228                 socket = 0;
 229
 230         rte_rwlock_init(&vq->iotlb_lock);
 231
 232         TAILQ_INIT(&vq->iotlb_list);
 233
 234         snprintf(pool_name, sizeof(pool_name), "iotlb_cache_%d_%d",
 235                         dev->vid, vq_index);
 236
 237         /* If already created, free it and recreate */
 238         vq->iotlb_pool = rte_mempool_lookup(pool_name);
 239         if (vq->iotlb_pool)
 240                 rte_mempool_free(vq->iotlb_pool);
 241
 242         vq->iotlb_pool = rte_mempool_create(pool_name,
 243                         IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
 244                         0, 0, NULL, NULL, NULL, socket,
 245                         MEMPOOL_F_NO_CACHE_ALIGN |
 246                         MEMPOOL_F_SP_PUT |
 247                         MEMPOOL_F_SC_GET);
 248         if (!vq->iotlb_pool) {
 249                 RTE_LOG(ERR, VHOST_CONFIG,
 250                                 "Failed to create IOTLB cache pool (%s)\n",
 251                                 pool_name);
 252                 return -1;
 253         }
 254
 255         vq->iotlb_cache_nr = 0;
 256
 257         return 0;
 258 }
 259