lib/vhost/iotlb.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2017 Red Hat, Inc.
   3  */
   4
   5 #ifdef RTE_LIBRTE_VHOST_NUMA
   6 #include <numaif.h>
   7 #endif
   8
   9 #include <rte_tailq.h>
  10
  11 #include "iotlb.h"
  12 #include "vhost.h"
  13
  14 struct vhost_iotlb_entry {
  15         TAILQ_ENTRY(vhost_iotlb_entry) next;
  16
  17         uint64_t iova;
  18         uint64_t uaddr;
  19         uint64_t size;
  20         uint8_t perm;
  21 };
  22
  23 #define IOTLB_CACHE_SIZE 2048
  24
  25 static void
  26 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
  27
  28 static void
  29 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
  30 {
  31         struct vhost_iotlb_entry *node, *temp_node;
  32
  33         rte_rwlock_write_lock(&vq->iotlb_pending_lock);
  34
  35         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
  36                 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
  37                 rte_mempool_put(vq->iotlb_pool, node);
  38         }
  39
  40         rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
  41 }
  42
  43 bool
  44 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
  45                                 uint8_t perm)
  46 {
  47         struct vhost_iotlb_entry *node;
  48         bool found = false;
  49
  50         rte_rwlock_read_lock(&vq->iotlb_pending_lock);
  51
  52         TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
  53                 if ((node->iova == iova) && (node->perm == perm)) {
  54                         found = true;
  55                         break;
  56                 }
  57         }
  58
  59         rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
  60
  61         return found;
  62 }
  63
  64 void
  65 vhost_user_iotlb_pending_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
  66                                 uint64_t iova, uint8_t perm)
  67 {
  68         struct vhost_iotlb_entry *node;
  69         int ret;
  70
  71         ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
  72         if (ret) {
  73                 VHOST_LOG_CONFIG(DEBUG,
  74                                 "(%s) IOTLB pool %s empty, clear entries for pending insertion\n",
  75                                 dev->ifname, vq->iotlb_pool->name);
  76                 if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
  77                         vhost_user_iotlb_pending_remove_all(vq);
  78                 else
  79                         vhost_user_iotlb_cache_random_evict(vq);
  80                 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
  81                 if (ret) {
  82                         VHOST_LOG_CONFIG(ERR,
  83                                         "(%s) IOTLB pool %s still empty, pending insertion failure\n",
  84                                         dev->ifname, vq->iotlb_pool->name);
  85                         return;
  86                 }
  87         }
  88
  89         node->iova = iova;
  90         node->perm = perm;
  91
  92         rte_rwlock_write_lock(&vq->iotlb_pending_lock);
  93
  94         TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
  95
  96         rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
  97 }
  98
  99 void
 100 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
 101                                 uint64_t iova, uint64_t size, uint8_t perm)
 102 {
 103         struct vhost_iotlb_entry *node, *temp_node;
 104
 105         rte_rwlock_write_lock(&vq->iotlb_pending_lock);
 106
 107         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next,
 108                                 temp_node) {
 109                 if (node->iova < iova)
 110                         continue;
 111                 if (node->iova >= iova + size)
 112                         continue;
 113                 if ((node->perm & perm) != node->perm)
 114                         continue;
 115                 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
 116                 rte_mempool_put(vq->iotlb_pool, node);
 117         }
 118
 119         rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
 120 }
 121
 122 static void
 123 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
 124 {
 125         struct vhost_iotlb_entry *node, *temp_node;
 126
 127         rte_rwlock_write_lock(&vq->iotlb_lock);
 128
 129         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
 130                 TAILQ_REMOVE(&vq->iotlb_list, node, next);
 131                 rte_mempool_put(vq->iotlb_pool, node);
 132         }
 133
 134         vq->iotlb_cache_nr = 0;
 135
 136         rte_rwlock_write_unlock(&vq->iotlb_lock);
 137 }
 138
 139 static void
 140 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
 141 {
 142         struct vhost_iotlb_entry *node, *temp_node;
 143         int entry_idx;
 144
 145         rte_rwlock_write_lock(&vq->iotlb_lock);
 146
 147         entry_idx = rte_rand() % vq->iotlb_cache_nr;
 148
 149         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
 150                 if (!entry_idx) {
 151                         TAILQ_REMOVE(&vq->iotlb_list, node, next);
 152                         rte_mempool_put(vq->iotlb_pool, node);
 153                         vq->iotlb_cache_nr--;
 154                         break;
 155                 }
 156                 entry_idx--;
 157         }
 158
 159         rte_rwlock_write_unlock(&vq->iotlb_lock);
 160 }
 161
 162 void
 163 vhost_user_iotlb_cache_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
 164                                 uint64_t iova, uint64_t uaddr,
 165                                 uint64_t size, uint8_t perm)
 166 {
 167         struct vhost_iotlb_entry *node, *new_node;
 168         int ret;
 169
 170         ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 171         if (ret) {
 172                 VHOST_LOG_CONFIG(DEBUG,
 173                                 "(%s) IOTLB pool %s empty, clear entries for cache insertion\n",
 174                                 dev->ifname, vq->iotlb_pool->name);
 175                 if (!TAILQ_EMPTY(&vq->iotlb_list))
 176                         vhost_user_iotlb_cache_random_evict(vq);
 177                 else
 178                         vhost_user_iotlb_pending_remove_all(vq);
 179                 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 180                 if (ret) {
 181                         VHOST_LOG_CONFIG(ERR,
 182                                         "(%s) IOTLB pool %s still empty, cache insertion failed\n",
 183                                         dev->ifname, vq->iotlb_pool->name);
 184                         return;
 185                 }
 186         }
 187
 188         new_node->iova = iova;
 189         new_node->uaddr = uaddr;
 190         new_node->size = size;
 191         new_node->perm = perm;
 192
 193         rte_rwlock_write_lock(&vq->iotlb_lock);
 194
 195         TAILQ_FOREACH(node, &vq->iotlb_list, next) {
 196                 /*
 197                  * Entries must be invalidated before being updated.
 198                  * So if iova already in list, assume identical.
 199                  */
 200                 if (node->iova == new_node->iova) {
 201                         rte_mempool_put(vq->iotlb_pool, new_node);
 202                         goto unlock;
 203                 } else if (node->iova > new_node->iova) {
 204                         TAILQ_INSERT_BEFORE(node, new_node, next);
 205                         vq->iotlb_cache_nr++;
 206                         goto unlock;
 207                 }
 208         }
 209
 210         TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
 211         vq->iotlb_cache_nr++;
 212
 213 unlock:
 214         vhost_user_iotlb_pending_remove(vq, iova, size, perm);
 215
 216         rte_rwlock_write_unlock(&vq->iotlb_lock);
 217
 218 }
 219
 220 void
 221 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
 222                                         uint64_t iova, uint64_t size)
 223 {
 224         struct vhost_iotlb_entry *node, *temp_node;
 225
 226         if (unlikely(!size))
 227                 return;
 228
 229         rte_rwlock_write_lock(&vq->iotlb_lock);
 230
 231         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
 232                 /* Sorted list */
 233                 if (unlikely(iova + size < node->iova))
 234                         break;
 235
 236                 if (iova < node->iova + node->size) {
 237                         TAILQ_REMOVE(&vq->iotlb_list, node, next);
 238                         rte_mempool_put(vq->iotlb_pool, node);
 239                         vq->iotlb_cache_nr--;
 240                 }
 241         }
 242
 243         rte_rwlock_write_unlock(&vq->iotlb_lock);
 244 }
 245
 246 uint64_t
 247 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
 248                                                 uint64_t *size, uint8_t perm)
 249 {
 250         struct vhost_iotlb_entry *node;
 251         uint64_t offset, vva = 0, mapped = 0;
 252
 253         if (unlikely(!*size))
 254                 goto out;
 255
 256         TAILQ_FOREACH(node, &vq->iotlb_list, next) {
 257                 /* List sorted by iova */
 258                 if (unlikely(iova < node->iova))
 259                         break;
 260
 261                 if (iova >= node->iova + node->size)
 262                         continue;
 263
 264                 if (unlikely((perm & node->perm) != perm)) {
 265                         vva = 0;
 266                         break;
 267                 }
 268
 269                 offset = iova - node->iova;
 270                 if (!vva)
 271                         vva = node->uaddr + offset;
 272
 273                 mapped += node->size - offset;
 274                 iova = node->iova + node->size;
 275
 276                 if (mapped >= *size)
 277                         break;
 278         }
 279
 280 out:
 281         /* Only part of the requested chunk is mapped */
 282         if (unlikely(mapped < *size))
 283                 *size = mapped;
 284
 285         return vva;
 286 }
 287
 288 void
 289 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
 290 {
 291         vhost_user_iotlb_cache_remove_all(vq);
 292         vhost_user_iotlb_pending_remove_all(vq);
 293 }
 294
 295 int
 296 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
 297 {
 298         char pool_name[RTE_MEMPOOL_NAMESIZE];
 299         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 300         int socket = 0;
 301
 302         if (vq->iotlb_pool) {
 303                 /*
 304                  * The cache has already been initialized,
 305                  * just drop all cached and pending entries.
 306                  */
 307                 vhost_user_iotlb_flush_all(vq);
 308         }
 309
 310 #ifdef RTE_LIBRTE_VHOST_NUMA
 311         if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
 312                 socket = 0;
 313 #endif
 314
 315         rte_rwlock_init(&vq->iotlb_lock);
 316         rte_rwlock_init(&vq->iotlb_pending_lock);
 317
 318         TAILQ_INIT(&vq->iotlb_list);
 319         TAILQ_INIT(&vq->iotlb_pending_list);
 320
 321         snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d",
 322                         getpid(), dev->vid, vq_index);
 323         VHOST_LOG_CONFIG(DEBUG, "(%s) IOTLB cache name: %s\n", dev->ifname, pool_name);
 324
 325         /* If already created, free it and recreate */
 326         vq->iotlb_pool = rte_mempool_lookup(pool_name);
 327         if (vq->iotlb_pool)
 328                 rte_mempool_free(vq->iotlb_pool);
 329
 330         vq->iotlb_pool = rte_mempool_create(pool_name,
 331                         IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
 332                         0, 0, NULL, NULL, NULL, socket,
 333                         RTE_MEMPOOL_F_NO_CACHE_ALIGN |
 334                         RTE_MEMPOOL_F_SP_PUT);
 335         if (!vq->iotlb_pool) {
 336                 VHOST_LOG_CONFIG(ERR, "(%s) Failed to create IOTLB cache pool %s\n",
 337                                 dev->ifname, pool_name);
 338                 return -1;
 339         }
 340
 341         vq->iotlb_cache_nr = 0;
 342
 343         return 0;
 344 }