X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=lib%2Flibrte_vhost%2Fiotlb.c;h=e0b67721b62380301c5d2ae56e13339342a1b721;hb=9253c34cfb9dd32faeb8e513f6d8fafd356a2e62;hp=fcfdd25d7cbce46d4ffede450988973f2b22bc80;hpb=d012d1f293f43e384b3117cd67d99f33aa0d21a9;p=dpdk.git diff --git a/lib/librte_vhost/iotlb.c b/lib/librte_vhost/iotlb.c index fcfdd25d7c..e0b67721b6 100644 --- a/lib/librte_vhost/iotlb.c +++ b/lib/librte_vhost/iotlb.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright (c) 2017 Red Hat, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017 Red Hat, Inc. */ #ifdef RTE_LIBRTE_VHOST_NUMA @@ -48,7 +20,99 @@ struct vhost_iotlb_entry { uint8_t perm; }; -#define IOTLB_CACHE_SIZE 1024 +#define IOTLB_CACHE_SIZE 2048 + +static void +vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq); + +static void +vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq) +{ + struct vhost_iotlb_entry *node, *temp_node; + + rte_rwlock_write_lock(&vq->iotlb_pending_lock); + + TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) { + TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); + rte_mempool_put(vq->iotlb_pool, node); + } + + rte_rwlock_write_unlock(&vq->iotlb_pending_lock); +} + +bool +vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova, + uint8_t perm) +{ + struct vhost_iotlb_entry *node; + bool found = false; + + rte_rwlock_read_lock(&vq->iotlb_pending_lock); + + TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) { + if ((node->iova == iova) && (node->perm == perm)) { + found = true; + break; + } + } + + rte_rwlock_read_unlock(&vq->iotlb_pending_lock); + + return found; +} + +void +vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq, + uint64_t iova, uint8_t perm) +{ + struct vhost_iotlb_entry *node; + int ret; + + ret = rte_mempool_get(vq->iotlb_pool, (void **)&node); + if (ret) { + VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n"); + if (!TAILQ_EMPTY(&vq->iotlb_pending_list)) + vhost_user_iotlb_pending_remove_all(vq); + else + vhost_user_iotlb_cache_random_evict(vq); + ret = rte_mempool_get(vq->iotlb_pool, (void **)&node); + if (ret) { + VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n"); + return; + } + } + + node->iova = iova; + node->perm = perm; + + rte_rwlock_write_lock(&vq->iotlb_pending_lock); + + TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next); + + rte_rwlock_write_unlock(&vq->iotlb_pending_lock); +} + +void +vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, + uint64_t iova, uint64_t size, uint8_t perm) +{ + struct vhost_iotlb_entry *node, *temp_node; + + rte_rwlock_write_lock(&vq->iotlb_pending_lock); + + TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) { + if (node->iova < iova) + continue; + if (node->iova >= iova + size) + continue; + if ((node->perm & perm) != node->perm) + continue; + TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); + rte_mempool_put(vq->iotlb_pool, node); + } + + rte_rwlock_write_unlock(&vq->iotlb_pending_lock); +} static void vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq) @@ -99,11 +163,14 @@ vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova, ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node); if (ret) { - RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, evict one entry\n"); - vhost_user_iotlb_cache_random_evict(vq); + VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n"); + if (!TAILQ_EMPTY(&vq->iotlb_list)) + vhost_user_iotlb_cache_random_evict(vq); + else + vhost_user_iotlb_pending_remove_all(vq); ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node); if (ret) { - RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n"); + VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n"); return; } } @@ -134,7 +201,10 @@ vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova, vq->iotlb_cache_nr++; unlock: + vhost_user_iotlb_pending_remove(vq, iova, size, perm); + rte_rwlock_write_unlock(&vq->iotlb_lock); + } void @@ -205,34 +275,42 @@ out: return vva; } +void +vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq) +{ + vhost_user_iotlb_cache_remove_all(vq); + vhost_user_iotlb_pending_remove_all(vq); +} + int vhost_user_iotlb_init(struct virtio_net *dev, int vq_index) { char pool_name[RTE_MEMPOOL_NAMESIZE]; struct vhost_virtqueue *vq = dev->virtqueue[vq_index]; - int ret = -1, socket; + int socket = 0; if (vq->iotlb_pool) { /* * The cache has already been initialized, - * just drop all entries + * just drop all cached and pending entries. */ - vhost_user_iotlb_cache_remove_all(vq); - return 0; + vhost_user_iotlb_flush_all(vq); } #ifdef RTE_LIBRTE_VHOST_NUMA - ret = get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR); -#endif - if (ret) + if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0) socket = 0; +#endif rte_rwlock_init(&vq->iotlb_lock); + rte_rwlock_init(&vq->iotlb_pending_lock); TAILQ_INIT(&vq->iotlb_list); + TAILQ_INIT(&vq->iotlb_pending_list); - snprintf(pool_name, sizeof(pool_name), "iotlb_cache_%d_%d", - dev->vid, vq_index); + snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d", + getpid(), dev->vid, vq_index); + VHOST_LOG_CONFIG(DEBUG, "IOTLB cache name: %s\n", pool_name); /* If already created, free it and recreate */ vq->iotlb_pool = rte_mempool_lookup(pool_name); @@ -243,10 +321,9 @@ vhost_user_iotlb_init(struct virtio_net *dev, int vq_index) IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0, 0, 0, NULL, NULL, NULL, socket, MEMPOOL_F_NO_CACHE_ALIGN | - MEMPOOL_F_SP_PUT | - MEMPOOL_F_SC_GET); + MEMPOOL_F_SP_PUT); if (!vq->iotlb_pool) { - RTE_LOG(ERR, VHOST_CONFIG, + VHOST_LOG_CONFIG(ERR, "Failed to create IOTLB cache pool (%s)\n", pool_name); return -1; @@ -256,4 +333,3 @@ vhost_user_iotlb_init(struct virtio_net *dev, int vq_index) return 0; } -