X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_eal%2Fcommon%2Feal_common_memzone.c;h=7c21aa921efcd7905708e8b764d57b2fd6f3377c;hb=402321cfca9b9f954ab960ddacc1c436b5b51869;hp=dae4ea074eb7505fa099810feb26a0ea7530ec70;hpb=af75078fece3615088e561357c1e97603e43a5fe;p=dpdk.git diff --git a/lib/librte_eal/common/eal_common_memzone.c b/lib/librte_eal/common/eal_common_memzone.c index dae4ea074e..7c21aa921e 100644 --- a/lib/librte_eal/common/eal_common_memzone.c +++ b/lib/librte_eal/common/eal_common_memzone.c @@ -1,36 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2012 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * version: DPDK.L.1.2.3-3 +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation */ #include @@ -45,59 +14,88 @@ #include #include #include -#include #include #include #include #include #include +#include +#include "malloc_heap.h" +#include "malloc_elem.h" #include "eal_private.h" +#include "eal_memcfg.h" -/* internal copy of free memory segments */ -static struct rte_memseg free_memseg[RTE_MAX_MEMSEG]; +static inline const struct rte_memzone * +memzone_lookup_thread_unsafe(const char *name) +{ + struct rte_mem_config *mcfg; + struct rte_fbarray *arr; + const struct rte_memzone *mz; + int i = 0; -/* pointer to last reserved memzone */ -static unsigned memzone_idx; + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + arr = &mcfg->memzones; -/* - * Return a pointer to a correctly filled memzone descriptor. If the - * allocation cannot be done, return NULL. - */ -const struct rte_memzone * -rte_memzone_reserve(const char *name, uint64_t len, int socket_id, - unsigned flags) -{ - return rte_memzone_reserve_aligned(name, - len, socket_id, flags, CACHE_LINE_SIZE); + /* + * the algorithm is not optimal (linear), but there are few + * zones and this function should be called at init only + */ + i = rte_fbarray_find_next_used(arr, 0); + while (i >= 0) { + mz = rte_fbarray_get(arr, i); + if (mz->addr != NULL && + !strncmp(name, mz->name, RTE_MEMZONE_NAMESIZE)) + return mz; + i = rte_fbarray_find_next_used(arr, i + 1); + } + return NULL; } -/* - * Return a pointer to a correctly filled memzone descriptor (with a - * specified alignment). If the allocation cannot be done, return NULL. - */ -const struct rte_memzone * -rte_memzone_reserve_aligned(const char *name, uint64_t len, - int socket_id, unsigned flags, unsigned align) +static const struct rte_memzone * +memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, + int socket_id, unsigned int flags, unsigned int align, + unsigned int bound) { - struct rte_config *config; - unsigned i = 0; - int memseg_idx = -1; - uint64_t requested_len; - uint64_t memseg_len = 0; - phys_addr_t memseg_physaddr; - void *memseg_addr; - uintptr_t addr_offset; - - /* if secondary processes return error */ - if (rte_eal_process_type() == RTE_PROC_SECONDARY){ - RTE_LOG(ERR, EAL, "%s(): Not allowed in secondary process\n", __func__); - rte_errno = E_RTE_SECONDARY; + struct rte_memzone *mz; + struct rte_mem_config *mcfg; + struct rte_fbarray *arr; + void *mz_addr; + size_t requested_len; + int mz_idx; + bool contig; + + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + arr = &mcfg->memzones; + + /* no more room in config */ + if (arr->count >= arr->len) { + RTE_LOG(ERR, EAL, + "%s(): Number of requested memzone segments exceeds RTE_MAX_MEMZONE\n", + __func__); + rte_errno = ENOSPC; + return NULL; + } + + if (strlen(name) > sizeof(mz->name) - 1) { + RTE_LOG(DEBUG, EAL, "%s(): memzone <%s>: name too long\n", + __func__, name); + rte_errno = ENAMETOOLONG; + return NULL; + } + + /* zone already exist */ + if ((memzone_lookup_thread_unsafe(name)) != NULL) { + RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n", + __func__, name); + rte_errno = EEXIST; return NULL; } /* if alignment is not a power of two */ - if (!rte_is_power_of_2(align)) { + if (align && !rte_is_power_of_2(align)) { RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__, align); rte_errno = EINVAL; @@ -105,231 +103,269 @@ rte_memzone_reserve_aligned(const char *name, uint64_t len, } /* alignment less than cache size is not allowed */ - if (align < CACHE_LINE_SIZE) - align = CACHE_LINE_SIZE; + if (align < RTE_CACHE_LINE_SIZE) + align = RTE_CACHE_LINE_SIZE; - /* get pointer to global configuration */ - config = rte_eal_get_configuration(); + /* align length on cache boundary. Check for overflow before doing so */ + if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) { + rte_errno = EINVAL; /* requested size too big */ + return NULL; + } - /* no more room in config */ - if (memzone_idx >= RTE_MAX_MEMZONE) { - RTE_LOG(ERR, EAL, "%s(): No more room in config\n", __func__); - rte_errno = ENOSPC; + len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE); + + /* save minimal requested length */ + requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len); + + /* check that boundary condition is valid */ + if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) { + rte_errno = EINVAL; return NULL; } - /* both sizes cannot be explicitly called for */ - if ((flags & RTE_MEMZONE_1GB) && (flags & RTE_MEMZONE_2MB)) { + if ((socket_id != SOCKET_ID_ANY) && socket_id < 0) { rte_errno = EINVAL; return NULL; } - /* zone already exist */ - if (rte_memzone_lookup(name) != NULL) { - RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n", - __func__, name); - rte_errno = EEXIST; + /* only set socket to SOCKET_ID_ANY if we aren't allocating for an + * external heap. + */ + if (!rte_eal_has_hugepages() && socket_id < RTE_MAX_NUMA_NODES) + socket_id = SOCKET_ID_ANY; + + contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0; + /* malloc only cares about size flags, remove contig flag from flags */ + flags &= ~RTE_MEMZONE_IOVA_CONTIG; + + if (len == 0 && bound == 0) { + /* no size constraints were placed, so use malloc elem len */ + requested_len = 0; + mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags, + align, contig); + } else { + if (len == 0) + requested_len = bound; + /* allocate memory on heap */ + mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id, + flags, align, bound, contig); + } + if (mz_addr == NULL) { + rte_errno = ENOMEM; return NULL; } - /* align length on cache boundary */ - len += CACHE_LINE_MASK; - len &= ~((uint64_t)CACHE_LINE_MASK); - - - - /* save requested length */ - requested_len = len; - - /* reserve extra space for future alignment */ - if (len) - len += align; - - /* find the smallest segment matching requirements */ - for (i = 0; i < RTE_MAX_MEMSEG; i++) { - - /* last segment */ - if (free_memseg[i].addr == NULL) - break; - - /* empty segment, skip it */ - if (free_memseg[i].len == 0) - continue; - - /* bad socket ID */ - if (socket_id != SOCKET_ID_ANY && - socket_id != free_memseg[i].socket_id) - continue; - - /* check len */ - if (len != 0 && len > free_memseg[i].len) - continue; - - /* check flags for hugepage sizes */ - if ((flags & RTE_MEMZONE_2MB) && - free_memseg[i].hugepage_sz == RTE_PGSIZE_1G ) - continue; - if ((flags & RTE_MEMZONE_1GB) && - free_memseg[i].hugepage_sz == RTE_PGSIZE_2M ) - continue; - - /* this segment is the best until now */ - if (memseg_idx == -1) { - memseg_idx = i; - memseg_len = free_memseg[i].len; - } - /* find the biggest contiguous zone */ - else if (len == 0) { - if (free_memseg[i].len > memseg_len) { - memseg_idx = i; - memseg_len = free_memseg[i].len; - } - } - /* - * find the smallest (we already checked that current - * zone length is > len - */ - else if (free_memseg[i].len < memseg_len) { - memseg_idx = i; - memseg_len = free_memseg[i].len; - } + struct malloc_elem *elem = malloc_elem_from_data(mz_addr); + + /* fill the zone in config */ + mz_idx = rte_fbarray_find_next_free(arr, 0); + + if (mz_idx < 0) { + mz = NULL; + } else { + rte_fbarray_set_used(arr, mz_idx); + mz = rte_fbarray_get(arr, mz_idx); } - /* no segment found */ - if (memseg_idx == -1) { - /* - * If RTE_MEMZONE_SIZE_HINT_ONLY flag is specified, - * try allocating again without the size parameter otherwise -fail. - */ - if ((flags & RTE_MEMZONE_SIZE_HINT_ONLY) && - ((flags & RTE_MEMZONE_1GB) || (flags & RTE_MEMZONE_2MB))) - return rte_memzone_reserve_aligned(name, len - align, - socket_id, 0, align); - - RTE_LOG(ERR, EAL, "%s(): No appropriate segment found\n", __func__); - rte_errno = ENOMEM; + if (mz == NULL) { + RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone\n", __func__); + malloc_heap_free(elem); + rte_errno = ENOSPC; return NULL; } - /* get offset needed to adjust alignment */ - addr_offset = (uintptr_t) RTE_PTR_SUB( - RTE_ALIGN_CEIL(free_memseg[memseg_idx].addr, (uintptr_t) align), - (uintptr_t) free_memseg[memseg_idx].addr); + strlcpy(mz->name, name, sizeof(mz->name)); + mz->iova = rte_malloc_virt2iova(mz_addr); + mz->addr = mz_addr; + mz->len = requested_len == 0 ? + elem->size - elem->pad - MALLOC_ELEM_OVERHEAD : + requested_len; + mz->hugepage_sz = elem->msl->page_sz; + mz->socket_id = elem->msl->socket_id; + mz->flags = 0; + + return mz; +} + +static const struct rte_memzone * +rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id, + unsigned int flags, unsigned int align, unsigned int bound) +{ + struct rte_mem_config *mcfg; + const struct rte_memzone *mz = NULL; - /* save aligned physical and virtual addresses */ - memseg_physaddr = free_memseg[memseg_idx].phys_addr + addr_offset; - memseg_addr = RTE_PTR_ADD(free_memseg[memseg_idx].addr, addr_offset); + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; - /* if we are looking for a biggest memzone */ - if (requested_len == 0) - requested_len = memseg_len - addr_offset; + rte_rwlock_write_lock(&mcfg->mlock); - /* set length to correct value */ - len = addr_offset + requested_len; + mz = memzone_reserve_aligned_thread_unsafe( + name, len, socket_id, flags, align, bound); - /* update our internal state */ - free_memseg[memseg_idx].len -= len; - free_memseg[memseg_idx].phys_addr += len; - free_memseg[memseg_idx].addr = - (char *)free_memseg[memseg_idx].addr + len; + rte_eal_trace_memzone_reserve(name, len, socket_id, flags, align, + bound, mz); - /* fill the zone in config */ - struct rte_memzone *mz = &config->mem_config->memzone[memzone_idx++]; - rte_snprintf(mz->name, sizeof(mz->name), "%s", name); - mz->phys_addr = memseg_physaddr; - mz->addr = memseg_addr; - mz->len = requested_len; - mz->hugepage_sz = free_memseg[memseg_idx].hugepage_sz; - mz->socket_id = free_memseg[memseg_idx].socket_id; - mz->flags = 0; + rte_rwlock_write_unlock(&mcfg->mlock); return mz; } /* - * Lookup for the memzone identified by the given name + * Return a pointer to a correctly filled memzone descriptor (with a + * specified alignment and boundary). If the allocation cannot be done, + * return NULL. */ const struct rte_memzone * -rte_memzone_lookup(const char *name) +rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id, + unsigned flags, unsigned align, unsigned bound) { - const struct rte_mem_config *mcfg; - unsigned i = 0; - - /* get pointer to global configuration */ - mcfg = rte_eal_get_configuration()->mem_config; + return rte_memzone_reserve_thread_safe(name, len, socket_id, flags, + align, bound); +} - /* - * the algorithm is not optimal (linear), but there are few - * zones and this function should be called at init only - */ - for (i = 0; i < RTE_MAX_MEMZONE && mcfg->memzone[i].addr != NULL; i++) { - if (!strncmp(name, mcfg->memzone[i].name, RTE_MEMZONE_NAMESIZE)) - return &mcfg->memzone[i]; - } - return NULL; +/* + * Return a pointer to a correctly filled memzone descriptor (with a + * specified alignment). If the allocation cannot be done, return NULL. + */ +const struct rte_memzone * +rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id, + unsigned flags, unsigned align) +{ + return rte_memzone_reserve_thread_safe(name, len, socket_id, flags, + align, 0); } -/* Dump all reserved memory zones on console */ -void -rte_memzone_dump(void) +/* + * Return a pointer to a correctly filled memzone descriptor. If the + * allocation cannot be done, return NULL. + */ +const struct rte_memzone * +rte_memzone_reserve(const char *name, size_t len, int socket_id, + unsigned flags) { - const struct rte_mem_config *mcfg; - unsigned i = 0; + return rte_memzone_reserve_thread_safe(name, len, socket_id, + flags, RTE_CACHE_LINE_SIZE, 0); +} - /* get pointer to global configuration */ +int +rte_memzone_free(const struct rte_memzone *mz) +{ + char name[RTE_MEMZONE_NAMESIZE]; + struct rte_mem_config *mcfg; + struct rte_fbarray *arr; + struct rte_memzone *found_mz; + int ret = 0; + void *addr = NULL; + unsigned idx; + + if (mz == NULL) + return -EINVAL; + + rte_strlcpy(name, mz->name, RTE_MEMZONE_NAMESIZE); mcfg = rte_eal_get_configuration()->mem_config; - - /* dump all zones */ - for (i=0; imemzone[i].addr == NULL) - break; - printf("name:<%s>, phys:0x%"PRIx64", len:0x%"PRIx64"" - ", virt:%p, socket_id:%"PRId32"\n", - mcfg->memzone[i].name, - mcfg->memzone[i].phys_addr, - mcfg->memzone[i].len, - mcfg->memzone[i].addr, - mcfg->memzone[i].socket_id); + arr = &mcfg->memzones; + + rte_rwlock_write_lock(&mcfg->mlock); + + idx = rte_fbarray_find_idx(arr, mz); + found_mz = rte_fbarray_get(arr, idx); + + if (found_mz == NULL) { + ret = -EINVAL; + } else if (found_mz->addr == NULL) { + RTE_LOG(ERR, EAL, "Memzone is not allocated\n"); + ret = -EINVAL; + } else { + addr = found_mz->addr; + memset(found_mz, 0, sizeof(*found_mz)); + rte_fbarray_set_free(arr, idx); } + + rte_rwlock_write_unlock(&mcfg->mlock); + + if (addr != NULL) + rte_free(addr); + + rte_eal_trace_memzone_free(name, addr, ret); + return ret; } /* - * called by init: modify the free memseg list to have cache-aligned - * addresses and cache-aligned lengths + * Lookup for the memzone identified by the given name */ -static int -memseg_sanitize(struct rte_memseg *memseg) +const struct rte_memzone * +rte_memzone_lookup(const char *name) { - unsigned phys_align; - unsigned virt_align; - unsigned off; + struct rte_mem_config *mcfg; + const struct rte_memzone *memzone = NULL; - phys_align = memseg->phys_addr & CACHE_LINE_MASK; - virt_align = (unsigned long)memseg->addr & CACHE_LINE_MASK; + mcfg = rte_eal_get_configuration()->mem_config; - /* - * sanity check: phys_addr and addr must have the same - * alignment - */ - if (phys_align != virt_align) - return -1; + rte_rwlock_read_lock(&mcfg->mlock); - /* memseg is really too small, don't bother with it */ - if (memseg->len < (2 * CACHE_LINE_SIZE)) { - memseg->len = 0; - return 0; - } + memzone = memzone_lookup_thread_unsafe(name); - /* align start address */ - off = (CACHE_LINE_SIZE - phys_align) & CACHE_LINE_MASK; - memseg->phys_addr += off; - memseg->addr = (char *)memseg->addr + off; - memseg->len -= off; + rte_rwlock_read_unlock(&mcfg->mlock); - /* align end address */ - memseg->len &= ~((uint64_t)CACHE_LINE_MASK); + rte_eal_trace_memzone_lookup(name, memzone); + return memzone; +} - return 0; +static void +dump_memzone(const struct rte_memzone *mz, void *arg) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + struct rte_memseg_list *msl = NULL; + void *cur_addr, *mz_end; + struct rte_memseg *ms; + int mz_idx, ms_idx; + size_t page_sz; + FILE *f = arg; + + mz_idx = rte_fbarray_find_idx(&mcfg->memzones, mz); + + fprintf(f, "Zone %u: name:<%s>, len:0x%zx, virt:%p, " + "socket_id:%"PRId32", flags:%"PRIx32"\n", + mz_idx, + mz->name, + mz->len, + mz->addr, + mz->socket_id, + mz->flags); + + /* go through each page occupied by this memzone */ + msl = rte_mem_virt2memseg_list(mz->addr); + if (!msl) { + RTE_LOG(DEBUG, EAL, "Skipping bad memzone\n"); + return; + } + page_sz = (size_t)mz->hugepage_sz; + cur_addr = RTE_PTR_ALIGN_FLOOR(mz->addr, page_sz); + mz_end = RTE_PTR_ADD(cur_addr, mz->len); + + fprintf(f, "physical segments used:\n"); + ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz; + ms = rte_fbarray_get(&msl->memseg_arr, ms_idx); + + do { + fprintf(f, " addr: %p iova: 0x%" PRIx64 " " + "len: 0x%zx " + "pagesz: 0x%zx\n", + cur_addr, ms->iova, ms->len, page_sz); + + /* advance VA to next page */ + cur_addr = RTE_PTR_ADD(cur_addr, page_sz); + + /* memzones occupy contiguous segments */ + ++ms; + } while (cur_addr < mz_end); +} + +/* Dump all reserved memory zones on console */ +void +rte_memzone_dump(FILE *f) +{ + rte_memzone_walk(dump_memzone, f); } /* @@ -338,39 +374,47 @@ memseg_sanitize(struct rte_memseg *memseg) int rte_eal_memzone_init(void) { - struct rte_config *config; - const struct rte_memseg *memseg; - unsigned i = 0; - - /* secondary processes don't need to initialise anything */ - if (rte_eal_process_type() == RTE_PROC_SECONDARY) - return 0; + struct rte_mem_config *mcfg; + int ret = 0; /* get pointer to global configuration */ - config = rte_eal_get_configuration(); + mcfg = rte_eal_get_configuration()->mem_config; - memseg = rte_eal_get_physmem_layout(); - if (memseg == NULL) { - RTE_LOG(ERR, EAL, "%s(): Cannot get physical layout\n", __func__); - return -1; + rte_rwlock_write_lock(&mcfg->mlock); + + if (rte_eal_process_type() == RTE_PROC_PRIMARY && + rte_fbarray_init(&mcfg->memzones, "memzone", + RTE_MAX_MEMZONE, sizeof(struct rte_memzone))) { + RTE_LOG(ERR, EAL, "Cannot allocate memzone list\n"); + ret = -1; + } else if (rte_eal_process_type() == RTE_PROC_SECONDARY && + rte_fbarray_attach(&mcfg->memzones)) { + RTE_LOG(ERR, EAL, "Cannot attach to memzone list\n"); + ret = -1; } - /* duplicate the memsegs from config */ - memcpy(free_memseg, memseg, sizeof(free_memseg)); - - /* make all zones cache-aligned */ - for (i=0; imlock); - /* delete all zones */ - memzone_idx = 0; - memset(config->mem_config->memzone, 0, sizeof(config->mem_config->memzone)); + return ret; +} - return 0; +/* Walk all reserved memory zones */ +void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *), + void *arg) +{ + struct rte_mem_config *mcfg; + struct rte_fbarray *arr; + int i; + + mcfg = rte_eal_get_configuration()->mem_config; + arr = &mcfg->memzones; + + rte_rwlock_read_lock(&mcfg->mlock); + i = rte_fbarray_find_next_used(arr, 0); + while (i >= 0) { + struct rte_memzone *mz = rte_fbarray_get(arr, i); + (*func)(mz, arg); + i = rte_fbarray_find_next_used(arr, i + 1); + } + rte_rwlock_read_unlock(&mcfg->mlock); }