1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
12 #include <sys/queue.h>
15 #include <rte_memory.h>
16 #include <rte_memzone.h>
18 #include <rte_per_lcore.h>
19 #include <rte_errno.h>
20 #include <rte_string_fns.h>
21 #include <rte_common.h>
23 #include "malloc_heap.h"
24 #include "malloc_elem.h"
25 #include "eal_private.h"
26 #include "eal_memcfg.h"
28 static inline const struct rte_memzone *
29 memzone_lookup_thread_unsafe(const char *name)
31 struct rte_mem_config *mcfg;
32 struct rte_fbarray *arr;
33 const struct rte_memzone *mz;
36 /* get pointer to global configuration */
37 mcfg = rte_eal_get_configuration()->mem_config;
38 arr = &mcfg->memzones;
41 * the algorithm is not optimal (linear), but there are few
42 * zones and this function should be called at init only
44 i = rte_fbarray_find_next_used(arr, 0);
46 mz = rte_fbarray_get(arr, i);
47 if (mz->addr != NULL &&
48 !strncmp(name, mz->name, RTE_MEMZONE_NAMESIZE))
50 i = rte_fbarray_find_next_used(arr, i + 1);
55 static const struct rte_memzone *
56 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
57 int socket_id, unsigned int flags, unsigned int align,
60 struct rte_memzone *mz;
61 struct rte_mem_config *mcfg;
62 struct rte_fbarray *arr;
68 /* get pointer to global configuration */
69 mcfg = rte_eal_get_configuration()->mem_config;
70 arr = &mcfg->memzones;
72 /* no more room in config */
73 if (arr->count >= arr->len) {
74 RTE_LOG(ERR, EAL, "%s(): No more room in config\n", __func__);
79 if (strlen(name) > sizeof(mz->name) - 1) {
80 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s>: name too long\n",
82 rte_errno = ENAMETOOLONG;
86 /* zone already exist */
87 if ((memzone_lookup_thread_unsafe(name)) != NULL) {
88 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n",
94 /* if alignment is not a power of two */
95 if (align && !rte_is_power_of_2(align)) {
96 RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__,
102 /* alignment less than cache size is not allowed */
103 if (align < RTE_CACHE_LINE_SIZE)
104 align = RTE_CACHE_LINE_SIZE;
106 /* align length on cache boundary. Check for overflow before doing so */
107 if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) {
108 rte_errno = EINVAL; /* requested size too big */
112 len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE);
114 /* save minimal requested length */
115 requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len);
117 /* check that boundary condition is valid */
118 if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) {
123 if ((socket_id != SOCKET_ID_ANY) && socket_id < 0) {
128 /* only set socket to SOCKET_ID_ANY if we aren't allocating for an
131 if (!rte_eal_has_hugepages() && socket_id < RTE_MAX_NUMA_NODES)
132 socket_id = SOCKET_ID_ANY;
134 contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0;
135 /* malloc only cares about size flags, remove contig flag from flags */
136 flags &= ~RTE_MEMZONE_IOVA_CONTIG;
138 if (len == 0 && bound == 0) {
139 /* no size constraints were placed, so use malloc elem len */
141 mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags,
145 requested_len = bound;
146 /* allocate memory on heap */
147 mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id,
148 flags, align, bound, contig);
150 if (mz_addr == NULL) {
155 struct malloc_elem *elem = malloc_elem_from_data(mz_addr);
157 /* fill the zone in config */
158 mz_idx = rte_fbarray_find_next_free(arr, 0);
163 rte_fbarray_set_used(arr, mz_idx);
164 mz = rte_fbarray_get(arr, mz_idx);
168 RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone\n", __func__);
169 malloc_heap_free(elem);
174 strlcpy(mz->name, name, sizeof(mz->name));
175 mz->iova = rte_malloc_virt2iova(mz_addr);
177 mz->len = requested_len == 0 ?
178 elem->size - elem->pad - MALLOC_ELEM_OVERHEAD :
180 mz->hugepage_sz = elem->msl->page_sz;
181 mz->socket_id = elem->msl->socket_id;
187 static const struct rte_memzone *
188 rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
189 unsigned int flags, unsigned int align, unsigned int bound)
191 struct rte_mem_config *mcfg;
192 const struct rte_memzone *mz = NULL;
194 /* get pointer to global configuration */
195 mcfg = rte_eal_get_configuration()->mem_config;
197 rte_rwlock_write_lock(&mcfg->mlock);
199 mz = memzone_reserve_aligned_thread_unsafe(
200 name, len, socket_id, flags, align, bound);
202 rte_rwlock_write_unlock(&mcfg->mlock);
208 * Return a pointer to a correctly filled memzone descriptor (with a
209 * specified alignment and boundary). If the allocation cannot be done,
212 const struct rte_memzone *
213 rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
214 unsigned flags, unsigned align, unsigned bound)
216 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
221 * Return a pointer to a correctly filled memzone descriptor (with a
222 * specified alignment). If the allocation cannot be done, return NULL.
224 const struct rte_memzone *
225 rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
226 unsigned flags, unsigned align)
228 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
233 * Return a pointer to a correctly filled memzone descriptor. If the
234 * allocation cannot be done, return NULL.
236 const struct rte_memzone *
237 rte_memzone_reserve(const char *name, size_t len, int socket_id,
240 return rte_memzone_reserve_thread_safe(name, len, socket_id,
241 flags, RTE_CACHE_LINE_SIZE, 0);
245 rte_memzone_free(const struct rte_memzone *mz)
247 struct rte_mem_config *mcfg;
248 struct rte_fbarray *arr;
249 struct rte_memzone *found_mz;
257 mcfg = rte_eal_get_configuration()->mem_config;
258 arr = &mcfg->memzones;
260 rte_rwlock_write_lock(&mcfg->mlock);
262 idx = rte_fbarray_find_idx(arr, mz);
263 found_mz = rte_fbarray_get(arr, idx);
265 if (found_mz == NULL) {
267 } else if (found_mz->addr == NULL) {
268 RTE_LOG(ERR, EAL, "Memzone is not allocated\n");
271 addr = found_mz->addr;
272 memset(found_mz, 0, sizeof(*found_mz));
273 rte_fbarray_set_free(arr, idx);
276 rte_rwlock_write_unlock(&mcfg->mlock);
285 * Lookup for the memzone identified by the given name
287 const struct rte_memzone *
288 rte_memzone_lookup(const char *name)
290 struct rte_mem_config *mcfg;
291 const struct rte_memzone *memzone = NULL;
293 mcfg = rte_eal_get_configuration()->mem_config;
295 rte_rwlock_read_lock(&mcfg->mlock);
297 memzone = memzone_lookup_thread_unsafe(name);
299 rte_rwlock_read_unlock(&mcfg->mlock);
305 dump_memzone(const struct rte_memzone *mz, void *arg)
307 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
308 struct rte_memseg_list *msl = NULL;
309 void *cur_addr, *mz_end;
310 struct rte_memseg *ms;
315 mz_idx = rte_fbarray_find_idx(&mcfg->memzones, mz);
317 fprintf(f, "Zone %u: name:<%s>, len:0x%zx, virt:%p, "
318 "socket_id:%"PRId32", flags:%"PRIx32"\n",
326 /* go through each page occupied by this memzone */
327 msl = rte_mem_virt2memseg_list(mz->addr);
329 RTE_LOG(DEBUG, EAL, "Skipping bad memzone\n");
332 page_sz = (size_t)mz->hugepage_sz;
333 cur_addr = RTE_PTR_ALIGN_FLOOR(mz->addr, page_sz);
334 mz_end = RTE_PTR_ADD(cur_addr, mz->len);
336 fprintf(f, "physical segments used:\n");
337 ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz;
338 ms = rte_fbarray_get(&msl->memseg_arr, ms_idx);
341 fprintf(f, " addr: %p iova: 0x%" PRIx64 " "
344 cur_addr, ms->iova, ms->len, page_sz);
346 /* advance VA to next page */
347 cur_addr = RTE_PTR_ADD(cur_addr, page_sz);
349 /* memzones occupy contiguous segments */
351 } while (cur_addr < mz_end);
354 /* Dump all reserved memory zones on console */
356 rte_memzone_dump(FILE *f)
358 rte_memzone_walk(dump_memzone, f);
362 * Init the memzone subsystem
365 rte_eal_memzone_init(void)
367 struct rte_mem_config *mcfg;
370 /* get pointer to global configuration */
371 mcfg = rte_eal_get_configuration()->mem_config;
373 rte_rwlock_write_lock(&mcfg->mlock);
375 if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
376 rte_fbarray_init(&mcfg->memzones, "memzone",
377 RTE_MAX_MEMZONE, sizeof(struct rte_memzone))) {
378 RTE_LOG(ERR, EAL, "Cannot allocate memzone list\n");
380 } else if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
381 rte_fbarray_attach(&mcfg->memzones)) {
382 RTE_LOG(ERR, EAL, "Cannot attach to memzone list\n");
386 rte_rwlock_write_unlock(&mcfg->mlock);
391 /* Walk all reserved memory zones */
392 void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *),
395 struct rte_mem_config *mcfg;
396 struct rte_fbarray *arr;
399 mcfg = rte_eal_get_configuration()->mem_config;
400 arr = &mcfg->memzones;
402 rte_rwlock_read_lock(&mcfg->mlock);
403 i = rte_fbarray_find_next_used(arr, 0);
405 struct rte_memzone *mz = rte_fbarray_get(arr, i);
407 i = rte_fbarray_find_next_used(arr, i + 1);
409 rte_rwlock_read_unlock(&mcfg->mlock);