1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2017 Red Hat, Inc.
5 #ifdef RTE_LIBRTE_VHOST_NUMA
14 struct vhost_iotlb_entry {
15 TAILQ_ENTRY(vhost_iotlb_entry) next;
23 #define IOTLB_CACHE_SIZE 2048
26 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
29 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
31 struct vhost_iotlb_entry *node, *temp_node;
33 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
35 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
36 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
37 rte_mempool_put(vq->iotlb_pool, node);
40 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
44 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
47 struct vhost_iotlb_entry *node;
50 rte_rwlock_read_lock(&vq->iotlb_pending_lock);
52 TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
53 if ((node->iova == iova) && (node->perm == perm)) {
59 rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
65 vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
66 uint64_t iova, uint8_t perm)
68 struct vhost_iotlb_entry *node;
71 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
73 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
74 if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
75 vhost_user_iotlb_pending_remove_all(vq);
77 vhost_user_iotlb_cache_random_evict(vq);
78 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
80 VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
88 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
90 TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
92 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
96 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
97 uint64_t iova, uint64_t size, uint8_t perm)
99 struct vhost_iotlb_entry *node, *temp_node;
101 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
103 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next,
105 if (node->iova < iova)
107 if (node->iova >= iova + size)
109 if ((node->perm & perm) != node->perm)
111 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
112 rte_mempool_put(vq->iotlb_pool, node);
115 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
119 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
121 struct vhost_iotlb_entry *node, *temp_node;
123 rte_rwlock_write_lock(&vq->iotlb_lock);
125 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
126 TAILQ_REMOVE(&vq->iotlb_list, node, next);
127 rte_mempool_put(vq->iotlb_pool, node);
130 vq->iotlb_cache_nr = 0;
132 rte_rwlock_write_unlock(&vq->iotlb_lock);
136 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
138 struct vhost_iotlb_entry *node, *temp_node;
141 rte_rwlock_write_lock(&vq->iotlb_lock);
143 entry_idx = rte_rand() % vq->iotlb_cache_nr;
145 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
147 TAILQ_REMOVE(&vq->iotlb_list, node, next);
148 rte_mempool_put(vq->iotlb_pool, node);
149 vq->iotlb_cache_nr--;
155 rte_rwlock_write_unlock(&vq->iotlb_lock);
159 vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
160 uint64_t uaddr, uint64_t size, uint8_t perm)
162 struct vhost_iotlb_entry *node, *new_node;
165 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
167 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
168 if (!TAILQ_EMPTY(&vq->iotlb_list))
169 vhost_user_iotlb_cache_random_evict(vq);
171 vhost_user_iotlb_pending_remove_all(vq);
172 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
174 VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
179 new_node->iova = iova;
180 new_node->uaddr = uaddr;
181 new_node->size = size;
182 new_node->perm = perm;
184 rte_rwlock_write_lock(&vq->iotlb_lock);
186 TAILQ_FOREACH(node, &vq->iotlb_list, next) {
188 * Entries must be invalidated before being updated.
189 * So if iova already in list, assume identical.
191 if (node->iova == new_node->iova) {
192 rte_mempool_put(vq->iotlb_pool, new_node);
194 } else if (node->iova > new_node->iova) {
195 TAILQ_INSERT_BEFORE(node, new_node, next);
196 vq->iotlb_cache_nr++;
201 TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
202 vq->iotlb_cache_nr++;
205 vhost_user_iotlb_pending_remove(vq, iova, size, perm);
207 rte_rwlock_write_unlock(&vq->iotlb_lock);
212 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
213 uint64_t iova, uint64_t size)
215 struct vhost_iotlb_entry *node, *temp_node;
220 rte_rwlock_write_lock(&vq->iotlb_lock);
222 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
224 if (unlikely(iova + size < node->iova))
227 if (iova < node->iova + node->size) {
228 TAILQ_REMOVE(&vq->iotlb_list, node, next);
229 rte_mempool_put(vq->iotlb_pool, node);
230 vq->iotlb_cache_nr--;
234 rte_rwlock_write_unlock(&vq->iotlb_lock);
238 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
239 uint64_t *size, uint8_t perm)
241 struct vhost_iotlb_entry *node;
242 uint64_t offset, vva = 0, mapped = 0;
244 if (unlikely(!*size))
247 TAILQ_FOREACH(node, &vq->iotlb_list, next) {
248 /* List sorted by iova */
249 if (unlikely(iova < node->iova))
252 if (iova >= node->iova + node->size)
255 if (unlikely((perm & node->perm) != perm)) {
260 offset = iova - node->iova;
262 vva = node->uaddr + offset;
264 mapped += node->size - offset;
265 iova = node->iova + node->size;
272 /* Only part of the requested chunk is mapped */
273 if (unlikely(mapped < *size))
280 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
282 vhost_user_iotlb_cache_remove_all(vq);
283 vhost_user_iotlb_pending_remove_all(vq);
287 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
289 char pool_name[RTE_MEMPOOL_NAMESIZE];
290 struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
293 if (vq->iotlb_pool) {
295 * The cache has already been initialized,
296 * just drop all cached and pending entries.
298 vhost_user_iotlb_flush_all(vq);
301 #ifdef RTE_LIBRTE_VHOST_NUMA
302 if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
306 rte_rwlock_init(&vq->iotlb_lock);
307 rte_rwlock_init(&vq->iotlb_pending_lock);
309 TAILQ_INIT(&vq->iotlb_list);
310 TAILQ_INIT(&vq->iotlb_pending_list);
312 snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d",
313 getpid(), dev->vid, vq_index);
314 VHOST_LOG_CONFIG(DEBUG, "IOTLB cache name: %s\n", pool_name);
316 /* If already created, free it and recreate */
317 vq->iotlb_pool = rte_mempool_lookup(pool_name);
319 rte_mempool_free(vq->iotlb_pool);
321 vq->iotlb_pool = rte_mempool_create(pool_name,
322 IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
323 0, 0, NULL, NULL, NULL, socket,
324 RTE_MEMPOOL_F_NO_CACHE_ALIGN |
325 RTE_MEMPOOL_F_SP_PUT);
326 if (!vq->iotlb_pool) {
327 VHOST_LOG_CONFIG(ERR,
328 "Failed to create IOTLB cache pool (%s)\n",
333 vq->iotlb_cache_nr = 0;