1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
14 #include <sys/queue.h>
16 #include <rte_memory.h>
18 #include <rte_eal_memconfig.h>
19 #include <rte_errno.h>
22 #include "eal_private.h"
23 #include "eal_internal_cfg.h"
26 * Try to mmap *size bytes in /dev/zero. If it is successful, return the
27 * pointer to the mmap'd area and keep *size unmodified. Else, retry
28 * with a smaller zone: decrease *size by hugepage_sz until it reaches
29 * 0. In this case, return NULL. Note: this function returns an address
30 * which is a multiple of hugepage size.
33 static uint64_t baseaddr_offset;
34 static uint64_t system_page_sz;
37 eal_get_virtual_area(void *requested_addr, size_t *size,
38 size_t page_sz, int flags, int mmap_flags)
40 bool addr_is_hint, allow_shrink, unmap, no_align;
42 void *mapped_addr, *aligned_addr;
44 if (system_page_sz == 0)
45 system_page_sz = sysconf(_SC_PAGESIZE);
47 mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
49 RTE_LOG(DEBUG, EAL, "Ask a virtual area of 0x%zx bytes\n", *size);
51 addr_is_hint = (flags & EAL_VIRTUAL_AREA_ADDR_IS_HINT) > 0;
52 allow_shrink = (flags & EAL_VIRTUAL_AREA_ALLOW_SHRINK) > 0;
53 unmap = (flags & EAL_VIRTUAL_AREA_UNMAP) > 0;
55 if (requested_addr == NULL && internal_config.base_virtaddr != 0) {
56 requested_addr = (void *) (internal_config.base_virtaddr +
57 (size_t)baseaddr_offset);
58 requested_addr = RTE_PTR_ALIGN(requested_addr, page_sz);
62 /* if requested address is not aligned by page size, or if requested
63 * address is NULL, add page size to requested length as we may get an
64 * address that's aligned by system page size, which can be smaller than
65 * our requested page size. additionally, we shouldn't try to align if
66 * system page size is the same as requested page size.
68 no_align = (requested_addr != NULL &&
69 ((uintptr_t)requested_addr & (page_sz - 1)) == 0) ||
70 page_sz == system_page_sz;
73 map_sz = no_align ? *size : *size + page_sz;
75 mapped_addr = mmap(requested_addr, map_sz, PROT_READ,
77 if (mapped_addr == MAP_FAILED && allow_shrink)
79 } while (allow_shrink && mapped_addr == MAP_FAILED && *size > 0);
81 /* align resulting address - if map failed, we will ignore the value
82 * anyway, so no need to add additional checks.
84 aligned_addr = no_align ? mapped_addr :
85 RTE_PTR_ALIGN(mapped_addr, page_sz);
88 RTE_LOG(ERR, EAL, "Cannot get a virtual area of any size: %s\n",
92 } else if (mapped_addr == MAP_FAILED) {
93 RTE_LOG(ERR, EAL, "Cannot get a virtual area: %s\n",
95 /* pass errno up the call chain */
98 } else if (requested_addr != NULL && !addr_is_hint &&
99 aligned_addr != requested_addr) {
100 RTE_LOG(ERR, EAL, "Cannot get a virtual area at requested address: %p (got %p)\n",
101 requested_addr, aligned_addr);
102 munmap(mapped_addr, map_sz);
103 rte_errno = EADDRNOTAVAIL;
105 } else if (requested_addr != NULL && addr_is_hint &&
106 aligned_addr != requested_addr) {
107 RTE_LOG(WARNING, EAL, "WARNING! Base virtual address hint (%p != %p) not respected!\n",
108 requested_addr, aligned_addr);
109 RTE_LOG(WARNING, EAL, " This may cause issues with mapping memory into secondary processes\n");
113 munmap(mapped_addr, map_sz);
115 RTE_LOG(DEBUG, EAL, "Virtual area found at %p (size = 0x%zx)\n",
116 aligned_addr, *size);
118 baseaddr_offset += *size;
124 * Return a pointer to a read-only table of struct rte_physmem_desc
125 * elements, containing the layout of all addressable physical
126 * memory. The last element of the table contains a NULL address.
128 const struct rte_memseg *
129 rte_eal_get_physmem_layout(void)
131 return rte_eal_get_configuration()->mem_config->memseg;
139 find_virt(const struct rte_memseg *ms, void *arg)
141 struct virtiova *vi = arg;
142 if (vi->iova >= ms->iova && vi->iova < (ms->iova + ms->len)) {
143 size_t offset = vi->iova - ms->iova;
144 vi->virt = RTE_PTR_ADD(ms->addr, offset);
151 __rte_experimental void *
152 rte_mem_iova2virt(rte_iova_t iova)
156 memset(&vi, 0, sizeof(vi));
159 rte_memseg_walk(find_virt, &vi);
165 physmem_size(const struct rte_memseg *ms, void *arg)
167 uint64_t *total_len = arg;
169 *total_len += ms->len;
174 /* get the total size of memory */
176 rte_eal_get_physmem_size(void)
178 uint64_t total_len = 0;
180 rte_memseg_walk(physmem_size, &total_len);
186 dump_memseg(const struct rte_memseg *ms, void *arg)
188 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
189 int i = ms - mcfg->memseg;
192 if (i < 0 || i >= RTE_MAX_MEMSEG)
195 fprintf(f, "Segment %u: IOVA:0x%"PRIx64", len:%zu, "
196 "virt:%p, socket_id:%"PRId32", "
197 "hugepage_sz:%"PRIu64", nchannel:%"PRIx32", "
198 "nrank:%"PRIx32"\n", i,
199 mcfg->memseg[i].iova,
201 mcfg->memseg[i].addr,
202 mcfg->memseg[i].socket_id,
203 mcfg->memseg[i].hugepage_sz,
204 mcfg->memseg[i].nchannel,
205 mcfg->memseg[i].nrank);
210 /* Dump the physical memory layout on console */
212 rte_dump_physmem_layout(FILE *f)
214 rte_memseg_walk(dump_memseg, f);
217 /* return the number of memory channels */
218 unsigned rte_memory_get_nchannel(void)
220 return rte_eal_get_configuration()->mem_config->nchannel;
223 /* return the number of memory rank */
224 unsigned rte_memory_get_nrank(void)
226 return rte_eal_get_configuration()->mem_config->nrank;
230 rte_eal_memdevice_init(void)
232 struct rte_config *config;
234 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
237 config = rte_eal_get_configuration();
238 config->mem_config->nchannel = internal_config.force_nchannel;
239 config->mem_config->nrank = internal_config.force_nrank;
244 /* Lock page in physical memory and prevent from swapping. */
246 rte_mem_lock_page(const void *virt)
248 unsigned long virtual = (unsigned long)virt;
249 int page_size = getpagesize();
250 unsigned long aligned = (virtual & ~(page_size - 1));
251 return mlock((void *)aligned, page_size);
254 int __rte_experimental
255 rte_memseg_walk(rte_memseg_walk_t func, void *arg)
257 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
260 for (i = 0; i < RTE_MAX_MEMSEG; i++) {
261 const struct rte_memseg *ms = &mcfg->memseg[i];
263 if (ms->addr == NULL)
275 int __rte_experimental
276 rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg)
278 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
281 for (i = 0; i < RTE_MAX_MEMSEG; i++) {
282 const struct rte_memseg *ms = &mcfg->memseg[i];
286 if (ms->addr == NULL)
289 end_addr = RTE_PTR_ADD(ms->addr, ms->len);
291 /* check how many more segments are contiguous to this one */
292 for (j = i + 1; j < RTE_MAX_MEMSEG; j++) {
293 const struct rte_memseg *next = &mcfg->memseg[j];
295 if (next->addr != end_addr)
298 end_addr = RTE_PTR_ADD(next->addr, next->len);
301 total_len = RTE_PTR_DIFF(end_addr, ms->addr);
303 ret = func(ms, total_len, arg);
312 /* init memory subsystem */
314 rte_eal_memory_init(void)
316 RTE_LOG(DEBUG, EAL, "Setting up physically contiguous memory...\n");
318 const int retval = rte_eal_process_type() == RTE_PROC_PRIMARY ?
319 rte_eal_hugepage_init() :
320 rte_eal_hugepage_attach();
324 if (internal_config.no_shconf == 0 && rte_eal_memdevice_init() < 0)