1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
12 #include <sys/queue.h>
15 #include <rte_memory.h>
16 #include <rte_memzone.h>
18 #include <rte_per_lcore.h>
19 #include <rte_errno.h>
20 #include <rte_string_fns.h>
21 #include <rte_common.h>
23 #include "malloc_heap.h"
24 #include "malloc_elem.h"
25 #include "eal_private.h"
26 #include "eal_memcfg.h"
28 static inline const struct rte_memzone *
29 memzone_lookup_thread_unsafe(const char *name)
31 struct rte_mem_config *mcfg;
32 struct rte_fbarray *arr;
33 const struct rte_memzone *mz;
36 /* get pointer to global configuration */
37 mcfg = rte_eal_get_configuration()->mem_config;
38 arr = &mcfg->memzones;
41 * the algorithm is not optimal (linear), but there are few
42 * zones and this function should be called at init only
44 i = rte_fbarray_find_next_used(arr, 0);
46 mz = rte_fbarray_get(arr, i);
47 if (mz->addr != NULL &&
48 !strncmp(name, mz->name, RTE_MEMZONE_NAMESIZE))
50 i = rte_fbarray_find_next_used(arr, i + 1);
55 static const struct rte_memzone *
56 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
57 int socket_id, unsigned int flags, unsigned int align,
60 struct rte_memzone *mz;
61 struct rte_mem_config *mcfg;
62 struct rte_fbarray *arr;
68 /* get pointer to global configuration */
69 mcfg = rte_eal_get_configuration()->mem_config;
70 arr = &mcfg->memzones;
72 /* no more room in config */
73 if (arr->count >= arr->len) {
75 "%s(): Number of requested memzone segments exceeds RTE_MAX_MEMZONE\n",
81 if (strlen(name) > sizeof(mz->name) - 1) {
82 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s>: name too long\n",
84 rte_errno = ENAMETOOLONG;
88 /* zone already exist */
89 if ((memzone_lookup_thread_unsafe(name)) != NULL) {
90 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n",
96 /* if alignment is not a power of two */
97 if (align && !rte_is_power_of_2(align)) {
98 RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__,
104 /* alignment less than cache size is not allowed */
105 if (align < RTE_CACHE_LINE_SIZE)
106 align = RTE_CACHE_LINE_SIZE;
108 /* align length on cache boundary. Check for overflow before doing so */
109 if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) {
110 rte_errno = EINVAL; /* requested size too big */
114 len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE);
116 /* save minimal requested length */
117 requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len);
119 /* check that boundary condition is valid */
120 if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) {
125 if ((socket_id != SOCKET_ID_ANY) && socket_id < 0) {
130 /* only set socket to SOCKET_ID_ANY if we aren't allocating for an
133 if (!rte_eal_has_hugepages() && socket_id < RTE_MAX_NUMA_NODES)
134 socket_id = SOCKET_ID_ANY;
136 contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0;
137 /* malloc only cares about size flags, remove contig flag from flags */
138 flags &= ~RTE_MEMZONE_IOVA_CONTIG;
140 if (len == 0 && bound == 0) {
141 /* no size constraints were placed, so use malloc elem len */
143 mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags,
147 requested_len = bound;
148 /* allocate memory on heap */
149 mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id,
150 flags, align, bound, contig);
152 if (mz_addr == NULL) {
157 struct malloc_elem *elem = malloc_elem_from_data(mz_addr);
159 /* fill the zone in config */
160 mz_idx = rte_fbarray_find_next_free(arr, 0);
165 rte_fbarray_set_used(arr, mz_idx);
166 mz = rte_fbarray_get(arr, mz_idx);
170 RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone\n", __func__);
171 malloc_heap_free(elem);
176 strlcpy(mz->name, name, sizeof(mz->name));
177 mz->iova = rte_malloc_virt2iova(mz_addr);
179 mz->len = requested_len == 0 ?
180 elem->size - elem->pad - MALLOC_ELEM_OVERHEAD :
182 mz->hugepage_sz = elem->msl->page_sz;
183 mz->socket_id = elem->msl->socket_id;
189 static const struct rte_memzone *
190 rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
191 unsigned int flags, unsigned int align, unsigned int bound)
193 struct rte_mem_config *mcfg;
194 const struct rte_memzone *mz = NULL;
196 /* get pointer to global configuration */
197 mcfg = rte_eal_get_configuration()->mem_config;
199 rte_rwlock_write_lock(&mcfg->mlock);
201 mz = memzone_reserve_aligned_thread_unsafe(
202 name, len, socket_id, flags, align, bound);
204 rte_rwlock_write_unlock(&mcfg->mlock);
210 * Return a pointer to a correctly filled memzone descriptor (with a
211 * specified alignment and boundary). If the allocation cannot be done,
214 const struct rte_memzone *
215 rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
216 unsigned flags, unsigned align, unsigned bound)
218 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
223 * Return a pointer to a correctly filled memzone descriptor (with a
224 * specified alignment). If the allocation cannot be done, return NULL.
226 const struct rte_memzone *
227 rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
228 unsigned flags, unsigned align)
230 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
235 * Return a pointer to a correctly filled memzone descriptor. If the
236 * allocation cannot be done, return NULL.
238 const struct rte_memzone *
239 rte_memzone_reserve(const char *name, size_t len, int socket_id,
242 return rte_memzone_reserve_thread_safe(name, len, socket_id,
243 flags, RTE_CACHE_LINE_SIZE, 0);
247 rte_memzone_free(const struct rte_memzone *mz)
249 struct rte_mem_config *mcfg;
250 struct rte_fbarray *arr;
251 struct rte_memzone *found_mz;
259 mcfg = rte_eal_get_configuration()->mem_config;
260 arr = &mcfg->memzones;
262 rte_rwlock_write_lock(&mcfg->mlock);
264 idx = rte_fbarray_find_idx(arr, mz);
265 found_mz = rte_fbarray_get(arr, idx);
267 if (found_mz == NULL) {
269 } else if (found_mz->addr == NULL) {
270 RTE_LOG(ERR, EAL, "Memzone is not allocated\n");
273 addr = found_mz->addr;
274 memset(found_mz, 0, sizeof(*found_mz));
275 rte_fbarray_set_free(arr, idx);
278 rte_rwlock_write_unlock(&mcfg->mlock);
287 * Lookup for the memzone identified by the given name
289 const struct rte_memzone *
290 rte_memzone_lookup(const char *name)
292 struct rte_mem_config *mcfg;
293 const struct rte_memzone *memzone = NULL;
295 mcfg = rte_eal_get_configuration()->mem_config;
297 rte_rwlock_read_lock(&mcfg->mlock);
299 memzone = memzone_lookup_thread_unsafe(name);
301 rte_rwlock_read_unlock(&mcfg->mlock);
307 dump_memzone(const struct rte_memzone *mz, void *arg)
309 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
310 struct rte_memseg_list *msl = NULL;
311 void *cur_addr, *mz_end;
312 struct rte_memseg *ms;
317 mz_idx = rte_fbarray_find_idx(&mcfg->memzones, mz);
319 fprintf(f, "Zone %u: name:<%s>, len:0x%zx, virt:%p, "
320 "socket_id:%"PRId32", flags:%"PRIx32"\n",
328 /* go through each page occupied by this memzone */
329 msl = rte_mem_virt2memseg_list(mz->addr);
331 RTE_LOG(DEBUG, EAL, "Skipping bad memzone\n");
334 page_sz = (size_t)mz->hugepage_sz;
335 cur_addr = RTE_PTR_ALIGN_FLOOR(mz->addr, page_sz);
336 mz_end = RTE_PTR_ADD(cur_addr, mz->len);
338 fprintf(f, "physical segments used:\n");
339 ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz;
340 ms = rte_fbarray_get(&msl->memseg_arr, ms_idx);
343 fprintf(f, " addr: %p iova: 0x%" PRIx64 " "
346 cur_addr, ms->iova, ms->len, page_sz);
348 /* advance VA to next page */
349 cur_addr = RTE_PTR_ADD(cur_addr, page_sz);
351 /* memzones occupy contiguous segments */
353 } while (cur_addr < mz_end);
356 /* Dump all reserved memory zones on console */
358 rte_memzone_dump(FILE *f)
360 rte_memzone_walk(dump_memzone, f);
364 * Init the memzone subsystem
367 rte_eal_memzone_init(void)
369 struct rte_mem_config *mcfg;
372 /* get pointer to global configuration */
373 mcfg = rte_eal_get_configuration()->mem_config;
375 rte_rwlock_write_lock(&mcfg->mlock);
377 if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
378 rte_fbarray_init(&mcfg->memzones, "memzone",
379 RTE_MAX_MEMZONE, sizeof(struct rte_memzone))) {
380 RTE_LOG(ERR, EAL, "Cannot allocate memzone list\n");
382 } else if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
383 rte_fbarray_attach(&mcfg->memzones)) {
384 RTE_LOG(ERR, EAL, "Cannot attach to memzone list\n");
388 rte_rwlock_write_unlock(&mcfg->mlock);
393 /* Walk all reserved memory zones */
394 void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *),
397 struct rte_mem_config *mcfg;
398 struct rte_fbarray *arr;
401 mcfg = rte_eal_get_configuration()->mem_config;
402 arr = &mcfg->memzones;
404 rte_rwlock_read_lock(&mcfg->mlock);
405 i = rte_fbarray_find_next_used(arr, 0);
407 struct rte_memzone *mz = rte_fbarray_get(arr, i);
409 i = rte_fbarray_find_next_used(arr, i + 1);
411 rte_rwlock_read_unlock(&mcfg->mlock);