1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
12 #include <sys/queue.h>
15 #include <rte_memory.h>
16 #include <rte_memzone.h>
18 #include <rte_eal_memconfig.h>
19 #include <rte_per_lcore.h>
20 #include <rte_errno.h>
21 #include <rte_string_fns.h>
22 #include <rte_common.h>
24 #include "malloc_heap.h"
25 #include "malloc_elem.h"
26 #include "eal_private.h"
28 static inline const struct rte_memzone *
29 memzone_lookup_thread_unsafe(const char *name)
31 struct rte_mem_config *mcfg;
32 struct rte_fbarray *arr;
33 const struct rte_memzone *mz;
36 /* get pointer to global configuration */
37 mcfg = rte_eal_get_configuration()->mem_config;
38 arr = &mcfg->memzones;
41 * the algorithm is not optimal (linear), but there are few
42 * zones and this function should be called at init only
44 i = rte_fbarray_find_next_used(arr, 0);
46 mz = rte_fbarray_get(arr, i);
47 if (mz->addr != NULL &&
48 !strncmp(name, mz->name, RTE_MEMZONE_NAMESIZE))
50 i = rte_fbarray_find_next_used(arr, i + 1);
55 static const struct rte_memzone *
56 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
57 int socket_id, unsigned int flags, unsigned int align,
60 struct rte_memzone *mz;
61 struct rte_mem_config *mcfg;
62 struct rte_fbarray *arr;
68 /* get pointer to global configuration */
69 mcfg = rte_eal_get_configuration()->mem_config;
70 arr = &mcfg->memzones;
72 /* no more room in config */
73 if (arr->count >= arr->len) {
74 RTE_LOG(ERR, EAL, "%s(): No more room in config\n", __func__);
79 if (strlen(name) > sizeof(mz->name) - 1) {
80 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s>: name too long\n",
82 rte_errno = ENAMETOOLONG;
86 /* zone already exist */
87 if ((memzone_lookup_thread_unsafe(name)) != NULL) {
88 RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n",
94 /* if alignment is not a power of two */
95 if (align && !rte_is_power_of_2(align)) {
96 RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__,
102 /* alignment less than cache size is not allowed */
103 if (align < RTE_CACHE_LINE_SIZE)
104 align = RTE_CACHE_LINE_SIZE;
106 /* align length on cache boundary. Check for overflow before doing so */
107 if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) {
108 rte_errno = EINVAL; /* requested size too big */
112 len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE);
114 /* save minimal requested length */
115 requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len);
117 /* check that boundary condition is valid */
118 if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) {
123 if ((socket_id != SOCKET_ID_ANY) &&
124 (socket_id >= RTE_MAX_NUMA_NODES || socket_id < 0)) {
129 if (!rte_eal_has_hugepages())
130 socket_id = SOCKET_ID_ANY;
132 contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0;
133 /* malloc only cares about size flags, remove contig flag from flags */
134 flags &= ~RTE_MEMZONE_IOVA_CONTIG;
136 if (len == 0 && bound == 0) {
137 /* no size constraints were placed, so use malloc elem len */
139 mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags,
143 requested_len = bound;
144 /* allocate memory on heap */
145 mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id,
146 flags, align, bound, contig);
148 if (mz_addr == NULL) {
153 struct malloc_elem *elem = malloc_elem_from_data(mz_addr);
155 /* fill the zone in config */
156 mz_idx = rte_fbarray_find_next_free(arr, 0);
161 rte_fbarray_set_used(arr, mz_idx);
162 mz = rte_fbarray_get(arr, mz_idx);
166 RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone\n", __func__);
167 malloc_heap_free(elem);
172 snprintf(mz->name, sizeof(mz->name), "%s", name);
173 mz->iova = rte_malloc_virt2iova(mz_addr);
175 mz->len = requested_len == 0 ?
176 elem->size - elem->pad - MALLOC_ELEM_OVERHEAD :
178 mz->hugepage_sz = elem->msl->page_sz;
179 mz->socket_id = elem->msl->socket_id;
185 static const struct rte_memzone *
186 rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
187 unsigned int flags, unsigned int align, unsigned int bound)
189 struct rte_mem_config *mcfg;
190 const struct rte_memzone *mz = NULL;
192 /* get pointer to global configuration */
193 mcfg = rte_eal_get_configuration()->mem_config;
195 rte_rwlock_write_lock(&mcfg->mlock);
197 mz = memzone_reserve_aligned_thread_unsafe(
198 name, len, socket_id, flags, align, bound);
200 rte_rwlock_write_unlock(&mcfg->mlock);
206 * Return a pointer to a correctly filled memzone descriptor (with a
207 * specified alignment and boundary). If the allocation cannot be done,
210 const struct rte_memzone *
211 rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
212 unsigned flags, unsigned align, unsigned bound)
214 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
219 * Return a pointer to a correctly filled memzone descriptor (with a
220 * specified alignment). If the allocation cannot be done, return NULL.
222 const struct rte_memzone *
223 rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
224 unsigned flags, unsigned align)
226 return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
231 * Return a pointer to a correctly filled memzone descriptor. If the
232 * allocation cannot be done, return NULL.
234 const struct rte_memzone *
235 rte_memzone_reserve(const char *name, size_t len, int socket_id,
238 return rte_memzone_reserve_thread_safe(name, len, socket_id,
239 flags, RTE_CACHE_LINE_SIZE, 0);
243 rte_memzone_free(const struct rte_memzone *mz)
245 struct rte_mem_config *mcfg;
246 struct rte_fbarray *arr;
247 struct rte_memzone *found_mz;
255 mcfg = rte_eal_get_configuration()->mem_config;
256 arr = &mcfg->memzones;
258 rte_rwlock_write_lock(&mcfg->mlock);
260 idx = rte_fbarray_find_idx(arr, mz);
261 found_mz = rte_fbarray_get(arr, idx);
263 if (found_mz == NULL) {
265 } else if (found_mz->addr == NULL) {
266 RTE_LOG(ERR, EAL, "Memzone is not allocated\n");
269 addr = found_mz->addr;
270 memset(found_mz, 0, sizeof(*found_mz));
271 rte_fbarray_set_free(arr, idx);
274 rte_rwlock_write_unlock(&mcfg->mlock);
283 * Lookup for the memzone identified by the given name
285 const struct rte_memzone *
286 rte_memzone_lookup(const char *name)
288 struct rte_mem_config *mcfg;
289 const struct rte_memzone *memzone = NULL;
291 mcfg = rte_eal_get_configuration()->mem_config;
293 rte_rwlock_read_lock(&mcfg->mlock);
295 memzone = memzone_lookup_thread_unsafe(name);
297 rte_rwlock_read_unlock(&mcfg->mlock);
303 dump_memzone(const struct rte_memzone *mz, void *arg)
305 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
306 struct rte_memseg_list *msl = NULL;
307 void *cur_addr, *mz_end;
308 struct rte_memseg *ms;
313 mz_idx = rte_fbarray_find_idx(&mcfg->memzones, mz);
315 fprintf(f, "Zone %u: name:<%s>, len:0x%zx, virt:%p, "
316 "socket_id:%"PRId32", flags:%"PRIx32"\n",
324 /* go through each page occupied by this memzone */
325 msl = rte_mem_virt2memseg_list(mz->addr);
327 RTE_LOG(DEBUG, EAL, "Skipping bad memzone\n");
330 page_sz = (size_t)mz->hugepage_sz;
331 cur_addr = RTE_PTR_ALIGN_FLOOR(mz->addr, page_sz);
332 mz_end = RTE_PTR_ADD(cur_addr, mz->len);
334 fprintf(f, "physical segments used:\n");
335 ms_idx = RTE_PTR_DIFF(mz->addr, msl->base_va) / page_sz;
336 ms = rte_fbarray_get(&msl->memseg_arr, ms_idx);
339 fprintf(f, " addr: %p iova: 0x%" PRIx64 " "
342 cur_addr, ms->iova, ms->len, page_sz);
344 /* advance VA to next page */
345 cur_addr = RTE_PTR_ADD(cur_addr, page_sz);
347 /* memzones occupy contiguous segments */
349 } while (cur_addr < mz_end);
352 /* Dump all reserved memory zones on console */
354 rte_memzone_dump(FILE *f)
356 rte_memzone_walk(dump_memzone, f);
360 * Init the memzone subsystem
363 rte_eal_memzone_init(void)
365 struct rte_mem_config *mcfg;
367 /* get pointer to global configuration */
368 mcfg = rte_eal_get_configuration()->mem_config;
370 rte_rwlock_write_lock(&mcfg->mlock);
372 if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
373 rte_fbarray_init(&mcfg->memzones, "memzone",
374 RTE_MAX_MEMZONE, sizeof(struct rte_memzone))) {
375 RTE_LOG(ERR, EAL, "Cannot allocate memzone list\n");
377 } else if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
378 rte_fbarray_attach(&mcfg->memzones)) {
379 RTE_LOG(ERR, EAL, "Cannot attach to memzone list\n");
380 rte_rwlock_write_unlock(&mcfg->mlock);
384 rte_rwlock_write_unlock(&mcfg->mlock);
389 /* Walk all reserved memory zones */
390 void rte_memzone_walk(void (*func)(const struct rte_memzone *, void *),
393 struct rte_mem_config *mcfg;
394 struct rte_fbarray *arr;
397 mcfg = rte_eal_get_configuration()->mem_config;
398 arr = &mcfg->memzones;
400 rte_rwlock_read_lock(&mcfg->mlock);
401 i = rte_fbarray_find_next_used(arr, 0);
403 struct rte_memzone *mz = rte_fbarray_get(arr, i);
405 i = rte_fbarray_find_next_used(arr, i + 1);
407 rte_rwlock_read_unlock(&mcfg->mlock);