1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2017 Red Hat, Inc.
5 #ifdef RTE_LIBRTE_VHOST_NUMA
14 struct vhost_iotlb_entry {
15 TAILQ_ENTRY(vhost_iotlb_entry) next;
23 #define IOTLB_CACHE_SIZE 2048
26 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
29 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
31 struct vhost_iotlb_entry *node, *temp_node;
33 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
35 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
36 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
37 rte_mempool_put(vq->iotlb_pool, node);
40 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
44 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
47 struct vhost_iotlb_entry *node;
50 rte_rwlock_read_lock(&vq->iotlb_pending_lock);
52 TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
53 if ((node->iova == iova) && (node->perm == perm)) {
59 rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
65 vhost_user_iotlb_pending_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
66 uint64_t iova, uint8_t perm)
68 struct vhost_iotlb_entry *node;
71 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
73 VHOST_LOG_CONFIG(DEBUG,
74 "(%s) IOTLB pool %s empty, clear entries for pending insertion\n",
75 dev->ifname, vq->iotlb_pool->name);
76 if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
77 vhost_user_iotlb_pending_remove_all(vq);
79 vhost_user_iotlb_cache_random_evict(vq);
80 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
83 "(%s) IOTLB pool %s still empty, pending insertion failure\n",
84 dev->ifname, vq->iotlb_pool->name);
92 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
94 TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
96 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
100 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
101 uint64_t iova, uint64_t size, uint8_t perm)
103 struct vhost_iotlb_entry *node, *temp_node;
105 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
107 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next,
109 if (node->iova < iova)
111 if (node->iova >= iova + size)
113 if ((node->perm & perm) != node->perm)
115 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
116 rte_mempool_put(vq->iotlb_pool, node);
119 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
123 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
125 struct vhost_iotlb_entry *node, *temp_node;
127 rte_rwlock_write_lock(&vq->iotlb_lock);
129 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
130 TAILQ_REMOVE(&vq->iotlb_list, node, next);
131 rte_mempool_put(vq->iotlb_pool, node);
134 vq->iotlb_cache_nr = 0;
136 rte_rwlock_write_unlock(&vq->iotlb_lock);
140 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
142 struct vhost_iotlb_entry *node, *temp_node;
145 rte_rwlock_write_lock(&vq->iotlb_lock);
147 entry_idx = rte_rand() % vq->iotlb_cache_nr;
149 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
151 TAILQ_REMOVE(&vq->iotlb_list, node, next);
152 rte_mempool_put(vq->iotlb_pool, node);
153 vq->iotlb_cache_nr--;
159 rte_rwlock_write_unlock(&vq->iotlb_lock);
163 vhost_user_iotlb_cache_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
164 uint64_t iova, uint64_t uaddr,
165 uint64_t size, uint8_t perm)
167 struct vhost_iotlb_entry *node, *new_node;
170 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
172 VHOST_LOG_CONFIG(DEBUG,
173 "(%s) IOTLB pool %s empty, clear entries for cache insertion\n",
174 dev->ifname, vq->iotlb_pool->name);
175 if (!TAILQ_EMPTY(&vq->iotlb_list))
176 vhost_user_iotlb_cache_random_evict(vq);
178 vhost_user_iotlb_pending_remove_all(vq);
179 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
181 VHOST_LOG_CONFIG(ERR,
182 "(%s) IOTLB pool %s still empty, cache insertion failed\n",
183 dev->ifname, vq->iotlb_pool->name);
188 new_node->iova = iova;
189 new_node->uaddr = uaddr;
190 new_node->size = size;
191 new_node->perm = perm;
193 rte_rwlock_write_lock(&vq->iotlb_lock);
195 TAILQ_FOREACH(node, &vq->iotlb_list, next) {
197 * Entries must be invalidated before being updated.
198 * So if iova already in list, assume identical.
200 if (node->iova == new_node->iova) {
201 rte_mempool_put(vq->iotlb_pool, new_node);
203 } else if (node->iova > new_node->iova) {
204 TAILQ_INSERT_BEFORE(node, new_node, next);
205 vq->iotlb_cache_nr++;
210 TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
211 vq->iotlb_cache_nr++;
214 vhost_user_iotlb_pending_remove(vq, iova, size, perm);
216 rte_rwlock_write_unlock(&vq->iotlb_lock);
221 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
222 uint64_t iova, uint64_t size)
224 struct vhost_iotlb_entry *node, *temp_node;
229 rte_rwlock_write_lock(&vq->iotlb_lock);
231 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
233 if (unlikely(iova + size < node->iova))
236 if (iova < node->iova + node->size) {
237 TAILQ_REMOVE(&vq->iotlb_list, node, next);
238 rte_mempool_put(vq->iotlb_pool, node);
239 vq->iotlb_cache_nr--;
243 rte_rwlock_write_unlock(&vq->iotlb_lock);
247 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
248 uint64_t *size, uint8_t perm)
250 struct vhost_iotlb_entry *node;
251 uint64_t offset, vva = 0, mapped = 0;
253 if (unlikely(!*size))
256 TAILQ_FOREACH(node, &vq->iotlb_list, next) {
257 /* List sorted by iova */
258 if (unlikely(iova < node->iova))
261 if (iova >= node->iova + node->size)
264 if (unlikely((perm & node->perm) != perm)) {
269 offset = iova - node->iova;
271 vva = node->uaddr + offset;
273 mapped += node->size - offset;
274 iova = node->iova + node->size;
281 /* Only part of the requested chunk is mapped */
282 if (unlikely(mapped < *size))
289 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
291 vhost_user_iotlb_cache_remove_all(vq);
292 vhost_user_iotlb_pending_remove_all(vq);
296 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
298 char pool_name[RTE_MEMPOOL_NAMESIZE];
299 struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
302 if (vq->iotlb_pool) {
304 * The cache has already been initialized,
305 * just drop all cached and pending entries.
307 vhost_user_iotlb_flush_all(vq);
310 #ifdef RTE_LIBRTE_VHOST_NUMA
311 if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
315 rte_rwlock_init(&vq->iotlb_lock);
316 rte_rwlock_init(&vq->iotlb_pending_lock);
318 TAILQ_INIT(&vq->iotlb_list);
319 TAILQ_INIT(&vq->iotlb_pending_list);
321 snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d",
322 getpid(), dev->vid, vq_index);
323 VHOST_LOG_CONFIG(DEBUG, "(%s) IOTLB cache name: %s\n", dev->ifname, pool_name);
325 /* If already created, free it and recreate */
326 vq->iotlb_pool = rte_mempool_lookup(pool_name);
327 rte_mempool_free(vq->iotlb_pool);
329 vq->iotlb_pool = rte_mempool_create(pool_name,
330 IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
331 0, 0, NULL, NULL, NULL, socket,
332 RTE_MEMPOOL_F_NO_CACHE_ALIGN |
333 RTE_MEMPOOL_F_SP_PUT);
334 if (!vq->iotlb_pool) {
335 VHOST_LOG_CONFIG(ERR, "(%s) Failed to create IOTLB cache pool %s\n",
336 dev->ifname, pool_name);
340 vq->iotlb_cache_nr = 0;