lib/vhost/iotlb.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2017 Red Hat, Inc.
   3  */
   4
   5 #ifdef RTE_LIBRTE_VHOST_NUMA
   6 #include <numaif.h>
   7 #endif
   8
   9 #include <rte_tailq.h>
  10
  11 #include "iotlb.h"
  12 #include "vhost.h"
  13
  14 struct vhost_iotlb_entry {
  15         TAILQ_ENTRY(vhost_iotlb_entry) next;
  16
  17         uint64_t iova;
  18         uint64_t uaddr;
  19         uint64_t size;
  20         uint8_t perm;
  21 };
  22
  23 #define IOTLB_CACHE_SIZE 2048
  24
  25 static void
  26 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
  27
  28 static void
  29 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
  30 {
  31         struct vhost_iotlb_entry *node, *temp_node;
  32
  33         rte_rwlock_write_lock(&vq->iotlb_pending_lock);
  34
  35         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
  36                 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
  37                 rte_mempool_put(vq->iotlb_pool, node);
  38         }
  39
  40         rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
  41 }
  42
  43 bool
  44 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
  45                                 uint8_t perm)
  46 {
  47         struct vhost_iotlb_entry *node;
  48         bool found = false;
  49
  50         rte_rwlock_read_lock(&vq->iotlb_pending_lock);
  51
  52         TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
  53                 if ((node->iova == iova) && (node->perm == perm)) {
  54                         found = true;
  55                         break;
  56                 }
  57         }
  58
  59         rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
  60
  61         return found;
  62 }
  63
  64 void
  65 vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
  66                                 uint64_t iova, uint8_t perm)
  67 {
  68         struct vhost_iotlb_entry *node;
  69         int ret;
  70
  71         ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
  72         if (ret) {
  73                 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
  74                 if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
  75                         vhost_user_iotlb_pending_remove_all(vq);
  76                 else
  77                         vhost_user_iotlb_cache_random_evict(vq);
  78                 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
  79                 if (ret) {
  80                         VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
  81                         return;
  82                 }
  83         }
  84
  85         node->iova = iova;
  86         node->perm = perm;
  87
  88         rte_rwlock_write_lock(&vq->iotlb_pending_lock);
  89
  90         TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
  91
  92         rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
  93 }
  94
  95 void
  96 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
  97                                 uint64_t iova, uint64_t size, uint8_t perm)
  98 {
  99         struct vhost_iotlb_entry *node, *temp_node;
 100
 101         rte_rwlock_write_lock(&vq->iotlb_pending_lock);
 102
 103         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next,
 104                                 temp_node) {
 105                 if (node->iova < iova)
 106                         continue;
 107                 if (node->iova >= iova + size)
 108                         continue;
 109                 if ((node->perm & perm) != node->perm)
 110                         continue;
 111                 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
 112                 rte_mempool_put(vq->iotlb_pool, node);
 113         }
 114
 115         rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
 116 }
 117
 118 static void
 119 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
 120 {
 121         struct vhost_iotlb_entry *node, *temp_node;
 122
 123         rte_rwlock_write_lock(&vq->iotlb_lock);
 124
 125         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
 126                 TAILQ_REMOVE(&vq->iotlb_list, node, next);
 127                 rte_mempool_put(vq->iotlb_pool, node);
 128         }
 129
 130         vq->iotlb_cache_nr = 0;
 131
 132         rte_rwlock_write_unlock(&vq->iotlb_lock);
 133 }
 134
 135 static void
 136 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
 137 {
 138         struct vhost_iotlb_entry *node, *temp_node;
 139         int entry_idx;
 140
 141         rte_rwlock_write_lock(&vq->iotlb_lock);
 142
 143         entry_idx = rte_rand() % vq->iotlb_cache_nr;
 144
 145         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
 146                 if (!entry_idx) {
 147                         TAILQ_REMOVE(&vq->iotlb_list, node, next);
 148                         rte_mempool_put(vq->iotlb_pool, node);
 149                         vq->iotlb_cache_nr--;
 150                         break;
 151                 }
 152                 entry_idx--;
 153         }
 154
 155         rte_rwlock_write_unlock(&vq->iotlb_lock);
 156 }
 157
 158 void
 159 vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
 160                                 uint64_t uaddr, uint64_t size, uint8_t perm)
 161 {
 162         struct vhost_iotlb_entry *node, *new_node;
 163         int ret;
 164
 165         ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 166         if (ret) {
 167                 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
 168                 if (!TAILQ_EMPTY(&vq->iotlb_list))
 169                         vhost_user_iotlb_cache_random_evict(vq);
 170                 else
 171                         vhost_user_iotlb_pending_remove_all(vq);
 172                 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 173                 if (ret) {
 174                         VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
 175                         return;
 176                 }
 177         }
 178
 179         new_node->iova = iova;
 180         new_node->uaddr = uaddr;
 181         new_node->size = size;
 182         new_node->perm = perm;
 183
 184         rte_rwlock_write_lock(&vq->iotlb_lock);
 185
 186         TAILQ_FOREACH(node, &vq->iotlb_list, next) {
 187                 /*
 188                  * Entries must be invalidated before being updated.
 189                  * So if iova already in list, assume identical.
 190                  */
 191                 if (node->iova == new_node->iova) {
 192                         rte_mempool_put(vq->iotlb_pool, new_node);
 193                         goto unlock;
 194                 } else if (node->iova > new_node->iova) {
 195                         TAILQ_INSERT_BEFORE(node, new_node, next);
 196                         vq->iotlb_cache_nr++;
 197                         goto unlock;
 198                 }
 199         }
 200
 201         TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
 202         vq->iotlb_cache_nr++;
 203
 204 unlock:
 205         vhost_user_iotlb_pending_remove(vq, iova, size, perm);
 206
 207         rte_rwlock_write_unlock(&vq->iotlb_lock);
 208
 209 }
 210
 211 void
 212 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
 213                                         uint64_t iova, uint64_t size)
 214 {
 215         struct vhost_iotlb_entry *node, *temp_node;
 216
 217         if (unlikely(!size))
 218                 return;
 219
 220         rte_rwlock_write_lock(&vq->iotlb_lock);
 221
 222         RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
 223                 /* Sorted list */
 224                 if (unlikely(iova + size < node->iova))
 225                         break;
 226
 227                 if (iova < node->iova + node->size) {
 228                         TAILQ_REMOVE(&vq->iotlb_list, node, next);
 229                         rte_mempool_put(vq->iotlb_pool, node);
 230                         vq->iotlb_cache_nr--;
 231                 }
 232         }
 233
 234         rte_rwlock_write_unlock(&vq->iotlb_lock);
 235 }
 236
 237 uint64_t
 238 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
 239                                                 uint64_t *size, uint8_t perm)
 240 {
 241         struct vhost_iotlb_entry *node;
 242         uint64_t offset, vva = 0, mapped = 0;
 243
 244         if (unlikely(!*size))
 245                 goto out;
 246
 247         TAILQ_FOREACH(node, &vq->iotlb_list, next) {
 248                 /* List sorted by iova */
 249                 if (unlikely(iova < node->iova))
 250                         break;
 251
 252                 if (iova >= node->iova + node->size)
 253                         continue;
 254
 255                 if (unlikely((perm & node->perm) != perm)) {
 256                         vva = 0;
 257                         break;
 258                 }
 259
 260                 offset = iova - node->iova;
 261                 if (!vva)
 262                         vva = node->uaddr + offset;
 263
 264                 mapped += node->size - offset;
 265                 iova = node->iova + node->size;
 266
 267                 if (mapped >= *size)
 268                         break;
 269         }
 270
 271 out:
 272         /* Only part of the requested chunk is mapped */
 273         if (unlikely(mapped < *size))
 274                 *size = mapped;
 275
 276         return vva;
 277 }
 278
 279 void
 280 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
 281 {
 282         vhost_user_iotlb_cache_remove_all(vq);
 283         vhost_user_iotlb_pending_remove_all(vq);
 284 }
 285
 286 int
 287 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
 288 {
 289         char pool_name[RTE_MEMPOOL_NAMESIZE];
 290         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 291         int socket = 0;
 292
 293         if (vq->iotlb_pool) {
 294                 /*
 295                  * The cache has already been initialized,
 296                  * just drop all cached and pending entries.
 297                  */
 298                 vhost_user_iotlb_flush_all(vq);
 299         }
 300
 301 #ifdef RTE_LIBRTE_VHOST_NUMA
 302         if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
 303                 socket = 0;
 304 #endif
 305
 306         rte_rwlock_init(&vq->iotlb_lock);
 307         rte_rwlock_init(&vq->iotlb_pending_lock);
 308
 309         TAILQ_INIT(&vq->iotlb_list);
 310         TAILQ_INIT(&vq->iotlb_pending_list);
 311
 312         snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d",
 313                         getpid(), dev->vid, vq_index);
 314         VHOST_LOG_CONFIG(DEBUG, "IOTLB cache name: %s\n", pool_name);
 315
 316         /* If already created, free it and recreate */
 317         vq->iotlb_pool = rte_mempool_lookup(pool_name);
 318         if (vq->iotlb_pool)
 319                 rte_mempool_free(vq->iotlb_pool);
 320
 321         vq->iotlb_pool = rte_mempool_create(pool_name,
 322                         IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
 323                         0, 0, NULL, NULL, NULL, socket,
 324                         RTE_MEMPOOL_F_NO_CACHE_ALIGN |
 325                         RTE_MEMPOOL_F_SP_PUT);
 326         if (!vq->iotlb_pool) {
 327                 VHOST_LOG_CONFIG(ERR,
 328                                 "Failed to create IOTLB cache pool (%s)\n",
 329                                 pool_name);
 330                 return -1;
 331         }
 332
 333         vq->iotlb_cache_nr = 0;
 334
 335         return 0;
 336 }