X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_eal%2Fcommon%2Feal_common_memzone.c;h=b5a5d7274f99f891e90427ff070f45030b063df9;hb=fdf20fa7bee9df9037116318a87080e1eb7e757e;hp=df707bfea869714708f7701e9f60278637afd572;hpb=dada9ef6edc59015b6674b5a95258787c71401b0;p=dpdk.git diff --git a/lib/librte_eal/common/eal_common_memzone.c b/lib/librte_eal/common/eal_common_memzone.c index df707bfea8..b5a5d7274f 100644 --- a/lib/librte_eal/common/eal_common_memzone.c +++ b/lib/librte_eal/common/eal_common_memzone.c @@ -1,35 +1,34 @@ /*- * BSD LICENSE - * - * Copyright(c) 2010-2012 Intel Corporation. All rights reserved. + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions * are met: - * - * * Redistributions of source code must retain the above copyright + * + * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * */ #include @@ -46,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -54,44 +54,104 @@ #include "eal_private.h" /* internal copy of free memory segments */ -static struct rte_memseg free_memseg[RTE_MAX_MEMSEG]; +static struct rte_memseg *free_memseg = NULL; -/* pointer to last reserved memzone */ -static unsigned memzone_idx; +static inline const struct rte_memzone * +memzone_lookup_thread_unsafe(const char *name) +{ + const struct rte_mem_config *mcfg; + unsigned i = 0; + + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + + /* + * the algorithm is not optimal (linear), but there are few + * zones and this function should be called at init only + */ + for (i = 0; i < RTE_MAX_MEMZONE && mcfg->memzone[i].addr != NULL; i++) { + if (!strncmp(name, mcfg->memzone[i].name, RTE_MEMZONE_NAMESIZE)) + return &mcfg->memzone[i]; + } + + return NULL; +} /* * Return a pointer to a correctly filled memzone descriptor. If the * allocation cannot be done, return NULL. */ const struct rte_memzone * -rte_memzone_reserve(const char *name, uint64_t len, int socket_id, +rte_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) { return rte_memzone_reserve_aligned(name, - len, socket_id, flags, CACHE_LINE_SIZE); + len, socket_id, flags, RTE_CACHE_LINE_SIZE); } /* - * Return a pointer to a correctly filled memzone descriptor (with a - * specified alignment). If the allocation cannot be done, return NULL. + * Helper function for memzone_reserve_aligned_thread_unsafe(). + * Calculate address offset from the start of the segment. + * Align offset in that way that it satisfy istart alignmnet and + * buffer of the requested length would not cross specified boundary. */ -const struct rte_memzone * -rte_memzone_reserve_aligned(const char *name, uint64_t len, - int socket_id, unsigned flags, unsigned align) +static inline phys_addr_t +align_phys_boundary(const struct rte_memseg *ms, size_t len, size_t align, + size_t bound) { - struct rte_config *config; + phys_addr_t addr_offset, bmask, end, start; + size_t step; + + step = RTE_MAX(align, bound); + bmask = ~((phys_addr_t)bound - 1); + + /* calculate offset to closest alignment */ + start = RTE_ALIGN_CEIL(ms->phys_addr, align); + addr_offset = start - ms->phys_addr; + + while (addr_offset + len < ms->len) { + + /* check, do we meet boundary condition */ + end = start + len - (len != 0); + if ((start & bmask) == (end & bmask)) + break; + + /* calculate next offset */ + start = RTE_ALIGN_CEIL(start + 1, step); + addr_offset = start - ms->phys_addr; + } + + return (addr_offset); +} + +static const struct rte_memzone * +memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, + int socket_id, unsigned flags, unsigned align, unsigned bound) +{ + struct rte_mem_config *mcfg; unsigned i = 0; int memseg_idx = -1; - uint64_t requested_len; - uint64_t memseg_len = 0; + uint64_t addr_offset, seg_offset = 0; + size_t requested_len; + size_t memseg_len = 0; phys_addr_t memseg_physaddr; void *memseg_addr; - uintptr_t addr_offset; - /* if secondary processes return error */ - if (rte_eal_process_type() == RTE_PROC_SECONDARY){ - RTE_LOG(ERR, EAL, "%s(): Not allowed in secondary process\n", __func__); - rte_errno = E_RTE_SECONDARY; + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + + /* no more room in config */ + if (mcfg->memzone_idx >= RTE_MAX_MEMZONE) { + RTE_LOG(ERR, EAL, "%s(): No more room in config\n", __func__); + rte_errno = ENOSPC; + return NULL; + } + + /* zone already exist */ + if ((memzone_lookup_thread_unsafe(name)) != NULL) { + RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n", + __func__, name); + rte_errno = EEXIST; return NULL; } @@ -104,49 +164,31 @@ rte_memzone_reserve_aligned(const char *name, uint64_t len, } /* alignment less than cache size is not allowed */ - if (align < CACHE_LINE_SIZE) - align = CACHE_LINE_SIZE; + if (align < RTE_CACHE_LINE_SIZE) + align = RTE_CACHE_LINE_SIZE; - /* get pointer to global configuration */ - config = rte_eal_get_configuration(); - /* no more room in config */ - if (memzone_idx >= RTE_MAX_MEMZONE) { - RTE_LOG(ERR, EAL, "%s(): No more room in config\n", __func__); - rte_errno = ENOSPC; + /* align length on cache boundary. Check for overflow before doing so */ + if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) { + rte_errno = EINVAL; /* requested size too big */ return NULL; } - /* both sizes cannot be explicitly called for */ - if ((flags & RTE_MEMZONE_1GB) && (flags & RTE_MEMZONE_2MB)) { - rte_errno = EINVAL; - return NULL; - } + len += RTE_CACHE_LINE_MASK; + len &= ~((size_t) RTE_CACHE_LINE_MASK); - /* zone already exist */ - if (rte_memzone_lookup(name) != NULL) { - RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n", - __func__, name); - rte_errno = EEXIST; + /* save minimal requested length */ + requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len); + + /* check that boundary condition is valid */ + if (bound != 0 && + (requested_len > bound || !rte_is_power_of_2(bound))) { + rte_errno = EINVAL; return NULL; } - /* align length on cache boundary */ - len += CACHE_LINE_MASK; - len &= ~((uint64_t)CACHE_LINE_MASK); - - - - /* save requested length */ - requested_len = len; - - /* reserve extra space for future alignment */ - if (len) - len += align; - /* find the smallest segment matching requirements */ for (i = 0; i < RTE_MAX_MEMSEG; i++) { - /* last segment */ if (free_memseg[i].addr == NULL) break; @@ -157,40 +199,59 @@ rte_memzone_reserve_aligned(const char *name, uint64_t len, /* bad socket ID */ if (socket_id != SOCKET_ID_ANY && + free_memseg[i].socket_id != SOCKET_ID_ANY && socket_id != free_memseg[i].socket_id) continue; + /* + * calculate offset to closest alignment that + * meets boundary conditions. + */ + addr_offset = align_phys_boundary(free_memseg + i, + requested_len, align, bound); + /* check len */ - if (len != 0 && len > free_memseg[i].len) + if ((requested_len + addr_offset) > free_memseg[i].len) continue; /* check flags for hugepage sizes */ if ((flags & RTE_MEMZONE_2MB) && - free_memseg[i].hugepage_sz == RTE_PGSIZE_1G ) + free_memseg[i].hugepage_sz == RTE_PGSIZE_1G) continue; if ((flags & RTE_MEMZONE_1GB) && - free_memseg[i].hugepage_sz == RTE_PGSIZE_2M ) + free_memseg[i].hugepage_sz == RTE_PGSIZE_2M) + continue; + if ((flags & RTE_MEMZONE_16MB) && + free_memseg[i].hugepage_sz == RTE_PGSIZE_16G) + continue; + if ((flags & RTE_MEMZONE_16GB) && + free_memseg[i].hugepage_sz == RTE_PGSIZE_16M) continue; /* this segment is the best until now */ if (memseg_idx == -1) { memseg_idx = i; memseg_len = free_memseg[i].len; + seg_offset = addr_offset; } /* find the biggest contiguous zone */ else if (len == 0) { if (free_memseg[i].len > memseg_len) { memseg_idx = i; memseg_len = free_memseg[i].len; + seg_offset = addr_offset; } } /* * find the smallest (we already checked that current * zone length is > len */ - else if (free_memseg[i].len < memseg_len) { + else if (free_memseg[i].len + align < memseg_len || + (free_memseg[i].len <= memseg_len + align && + addr_offset < seg_offset)) { memseg_idx = i; memseg_len = free_memseg[i].len; + seg_offset = addr_offset; } } @@ -201,30 +262,31 @@ rte_memzone_reserve_aligned(const char *name, uint64_t len, * try allocating again without the size parameter otherwise -fail. */ if ((flags & RTE_MEMZONE_SIZE_HINT_ONLY) && - ((flags & RTE_MEMZONE_1GB) || (flags & RTE_MEMZONE_2MB))) - return rte_memzone_reserve_aligned(name, len - align, - socket_id, 0, align); + ((flags & RTE_MEMZONE_1GB) || (flags & RTE_MEMZONE_2MB) + || (flags & RTE_MEMZONE_16MB) || (flags & RTE_MEMZONE_16GB))) + return memzone_reserve_aligned_thread_unsafe(name, + len, socket_id, 0, align, bound); - RTE_LOG(ERR, EAL, "%s(): No appropriate segment found\n", __func__); rte_errno = ENOMEM; return NULL; } - /* get offset needed to adjust alignment */ - addr_offset = (uintptr_t) RTE_PTR_SUB( - RTE_ALIGN_CEIL(free_memseg[memseg_idx].addr, (uintptr_t) align), - (uintptr_t) free_memseg[memseg_idx].addr); - /* save aligned physical and virtual addresses */ - memseg_physaddr = free_memseg[memseg_idx].phys_addr + addr_offset; - memseg_addr = RTE_PTR_ADD(free_memseg[memseg_idx].addr, addr_offset); + memseg_physaddr = free_memseg[memseg_idx].phys_addr + seg_offset; + memseg_addr = RTE_PTR_ADD(free_memseg[memseg_idx].addr, + (uintptr_t) seg_offset); /* if we are looking for a biggest memzone */ - if (requested_len == 0) - requested_len = memseg_len - addr_offset; + if (len == 0) { + if (bound == 0) + requested_len = memseg_len - seg_offset; + else + requested_len = RTE_ALIGN_CEIL(memseg_physaddr + 1, + bound) - memseg_physaddr; + } /* set length to correct value */ - len = addr_offset + requested_len; + len = (size_t)seg_offset + requested_len; /* update our internal state */ free_memseg[memseg_idx].len -= len; @@ -233,63 +295,128 @@ rte_memzone_reserve_aligned(const char *name, uint64_t len, (char *)free_memseg[memseg_idx].addr + len; /* fill the zone in config */ - struct rte_memzone *mz = &config->mem_config->memzone[memzone_idx++]; - rte_snprintf(mz->name, sizeof(mz->name), "%s", name); + struct rte_memzone *mz = &mcfg->memzone[mcfg->memzone_idx++]; + snprintf(mz->name, sizeof(mz->name), "%s", name); mz->phys_addr = memseg_physaddr; mz->addr = memseg_addr; mz->len = requested_len; mz->hugepage_sz = free_memseg[memseg_idx].hugepage_sz; mz->socket_id = free_memseg[memseg_idx].socket_id; mz->flags = 0; + mz->memseg_id = memseg_idx; return mz; } /* - * Lookup for the memzone identified by the given name + * Return a pointer to a correctly filled memzone descriptor (with a + * specified alignment). If the allocation cannot be done, return NULL. */ const struct rte_memzone * -rte_memzone_lookup(const char *name) +rte_memzone_reserve_aligned(const char *name, size_t len, + int socket_id, unsigned flags, unsigned align) { - const struct rte_mem_config *mcfg; - unsigned i = 0; + struct rte_mem_config *mcfg; + const struct rte_memzone *mz = NULL; + + /* both sizes cannot be explicitly called for */ + if (((flags & RTE_MEMZONE_1GB) && (flags & RTE_MEMZONE_2MB)) + || ((flags & RTE_MEMZONE_16MB) && (flags & RTE_MEMZONE_16GB))) { + rte_errno = EINVAL; + return NULL; + } /* get pointer to global configuration */ mcfg = rte_eal_get_configuration()->mem_config; - /* - * the algorithm is not optimal (linear), but there are few - * zones and this function should be called at init only - */ - for (i = 0; i < RTE_MAX_MEMZONE && mcfg->memzone[i].addr != NULL; i++) { - if (!strncmp(name, mcfg->memzone[i].name, RTE_MEMZONE_NAMESIZE)) - return &mcfg->memzone[i]; + rte_rwlock_write_lock(&mcfg->mlock); + + mz = memzone_reserve_aligned_thread_unsafe( + name, len, socket_id, flags, align, 0); + + rte_rwlock_write_unlock(&mcfg->mlock); + + return mz; +} + +/* + * Return a pointer to a correctly filled memzone descriptor (with a + * specified alignment and boundary). + * If the allocation cannot be done, return NULL. + */ +const struct rte_memzone * +rte_memzone_reserve_bounded(const char *name, size_t len, + int socket_id, unsigned flags, unsigned align, unsigned bound) +{ + struct rte_mem_config *mcfg; + const struct rte_memzone *mz = NULL; + + /* both sizes cannot be explicitly called for */ + if (((flags & RTE_MEMZONE_1GB) && (flags & RTE_MEMZONE_2MB)) + || ((flags & RTE_MEMZONE_16MB) && (flags & RTE_MEMZONE_16GB))) { + rte_errno = EINVAL; + return NULL; } - return NULL; + + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + + rte_rwlock_write_lock(&mcfg->mlock); + + mz = memzone_reserve_aligned_thread_unsafe( + name, len, socket_id, flags, align, bound); + + rte_rwlock_write_unlock(&mcfg->mlock); + + return mz; +} + + +/* + * Lookup for the memzone identified by the given name + */ +const struct rte_memzone * +rte_memzone_lookup(const char *name) +{ + struct rte_mem_config *mcfg; + const struct rte_memzone *memzone = NULL; + + mcfg = rte_eal_get_configuration()->mem_config; + + rte_rwlock_read_lock(&mcfg->mlock); + + memzone = memzone_lookup_thread_unsafe(name); + + rte_rwlock_read_unlock(&mcfg->mlock); + + return memzone; } /* Dump all reserved memory zones on console */ void -rte_memzone_dump(void) +rte_memzone_dump(FILE *f) { - const struct rte_mem_config *mcfg; + struct rte_mem_config *mcfg; unsigned i = 0; /* get pointer to global configuration */ mcfg = rte_eal_get_configuration()->mem_config; + rte_rwlock_read_lock(&mcfg->mlock); /* dump all zones */ for (i=0; imemzone[i].addr == NULL) break; - printf("name:<%s>, phys:0x%"PRIx64", len:0x%"PRIx64"" - ", virt:%p, socket_id:%"PRId32"\n", + fprintf(f, "Zone %u: name:<%s>, phys:0x%"PRIx64", len:0x%zx" + ", virt:%p, socket_id:%"PRId32", flags:%"PRIx32"\n", i, mcfg->memzone[i].name, mcfg->memzone[i].phys_addr, mcfg->memzone[i].len, mcfg->memzone[i].addr, - mcfg->memzone[i].socket_id); + mcfg->memzone[i].socket_id, + mcfg->memzone[i].flags); } + rte_rwlock_read_unlock(&mcfg->mlock); } /* @@ -303,8 +430,8 @@ memseg_sanitize(struct rte_memseg *memseg) unsigned virt_align; unsigned off; - phys_align = memseg->phys_addr & CACHE_LINE_MASK; - virt_align = (unsigned long)memseg->addr & CACHE_LINE_MASK; + phys_align = memseg->phys_addr & RTE_CACHE_LINE_MASK; + virt_align = (unsigned long)memseg->addr & RTE_CACHE_LINE_MASK; /* * sanity check: phys_addr and addr must have the same @@ -314,19 +441,19 @@ memseg_sanitize(struct rte_memseg *memseg) return -1; /* memseg is really too small, don't bother with it */ - if (memseg->len < (2 * CACHE_LINE_SIZE)) { + if (memseg->len < (2 * RTE_CACHE_LINE_SIZE)) { memseg->len = 0; return 0; } /* align start address */ - off = (CACHE_LINE_SIZE - phys_align) & CACHE_LINE_MASK; + off = (RTE_CACHE_LINE_SIZE - phys_align) & RTE_CACHE_LINE_MASK; memseg->phys_addr += off; memseg->addr = (char *)memseg->addr + off; memseg->len -= off; /* align end address */ - memseg->len &= ~((uint64_t)CACHE_LINE_MASK); + memseg->len &= ~((uint64_t)RTE_CACHE_LINE_MASK); return 0; } @@ -337,39 +464,70 @@ memseg_sanitize(struct rte_memseg *memseg) int rte_eal_memzone_init(void) { - struct rte_config *config; + struct rte_mem_config *mcfg; const struct rte_memseg *memseg; unsigned i = 0; + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + + /* mirror the runtime memsegs from config */ + free_memseg = mcfg->free_memseg; + /* secondary processes don't need to initialise anything */ if (rte_eal_process_type() == RTE_PROC_SECONDARY) return 0; - /* get pointer to global configuration */ - config = rte_eal_get_configuration(); - memseg = rte_eal_get_physmem_layout(); if (memseg == NULL) { RTE_LOG(ERR, EAL, "%s(): Cannot get physical layout\n", __func__); return -1; } - /* duplicate the memsegs from config */ - memcpy(free_memseg, memseg, sizeof(free_memseg)); + rte_rwlock_write_lock(&mcfg->mlock); + + /* fill in uninitialized free_memsegs */ + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if (memseg[i].addr == NULL) + break; + if (free_memseg[i].addr != NULL) + continue; + memcpy(&free_memseg[i], &memseg[i], sizeof(struct rte_memseg)); + } /* make all zones cache-aligned */ - for (i=0; imlock); return -1; } } /* delete all zones */ - memzone_idx = 0; - memset(config->mem_config->memzone, 0, sizeof(config->mem_config->memzone)); + mcfg->memzone_idx = 0; + memset(mcfg->memzone, 0, sizeof(mcfg->memzone)); + + rte_rwlock_write_unlock(&mcfg->mlock); return 0; } + +/* Walk all reserved memory zones */ +void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *), + void *arg) +{ + struct rte_mem_config *mcfg; + unsigned i; + + mcfg = rte_eal_get_configuration()->mem_config; + + rte_rwlock_read_lock(&mcfg->mlock); + for (i=0; imemzone[i].addr != NULL) + (*func)(&mcfg->memzone[i], arg); + } + rte_rwlock_read_unlock(&mcfg->mlock); +}