char segname[250]; /* as per manpage, limit is 249 bytes plus null */
int flags = RTE_MFD_HUGETLB | pagesz_flags(hi->hugepage_sz);
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
- if (internal_config.single_file_segments) {
+ if (internal_conf->single_file_segments) {
fd = fd_list[list_idx].memseg_list_fd;
if (fd < 0) {
unsigned int list_idx, unsigned int seg_idx)
{
int fd;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
/* for in-memory mode, we only make it here when we're sure we support
* memfd, and this is a special case.
*/
- if (internal_config.in_memory)
+ if (internal_conf->in_memory)
return get_seg_memfd(hi, list_idx, seg_idx);
- if (internal_config.single_file_segments) {
+ if (internal_conf->single_file_segments) {
/* create a hugepage file path */
eal_get_hugefile_path(path, buflen, hi->hugedir, list_idx);
fd = fd_list[list_idx].fds[seg_idx];
if (fd < 0) {
+ /* A primary process is the only one creating these
+ * files. If there is a leftover that was not cleaned
+ * by clear_hugedir(), we must *now* make sure to drop
+ * the file or we will remap old stuff while the rest
+ * of the code is built on the assumption that a new
+ * page is clean.
+ */
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
+ unlink(path) == -1 &&
+ errno != ENOENT) {
+ RTE_LOG(DEBUG, EAL, "%s(): could not remove '%s': %s\n",
+ __func__, path, strerror(errno));
+ return -1;
+ }
+
fd = open(path, O_CREAT | O_RDWR, 0600);
if (fd < 0) {
RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n",
static void
close_hugefile(int fd, char *path, int list_idx)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
/*
* primary process must unlink the file, but only when not in in-memory
* mode (as in that case there is no file to unlink).
*/
- if (!internal_config.in_memory &&
+ if (!internal_conf->in_memory &&
rte_eal_process_type() == RTE_PROC_PRIMARY &&
unlink(path))
RTE_LOG(ERR, EAL, "%s(): unlinking '%s' failed: %s\n",
/* in-memory mode is a special case, because we can be sure that
* fallocate() is supported.
*/
- if (internal_config.in_memory)
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
+ if (internal_conf->in_memory)
return resize_hugefile_in_memory(fd, fa_offset,
page_sz, grow);
size_t alloc_sz;
int flags;
void *new_addr;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
alloc_sz = hi->hugepage_sz;
/* these are checked at init, but code analyzers don't know that */
- if (internal_config.in_memory && !anonymous_hugepages_supported) {
+ if (internal_conf->in_memory && !anonymous_hugepages_supported) {
RTE_LOG(ERR, EAL, "Anonymous hugepages not supported, in-memory mode cannot allocate memory\n");
return -1;
}
- if (internal_config.in_memory && !memfd_create_supported &&
- internal_config.single_file_segments) {
+ if (internal_conf->in_memory && !memfd_create_supported &&
+ internal_conf->single_file_segments) {
RTE_LOG(ERR, EAL, "Single-file segments are not supported without memfd support\n");
return -1;
}
/* in-memory without memfd is a special case */
int mmap_flags;
- if (internal_config.in_memory && !memfd_create_supported) {
+ if (internal_conf->in_memory && !memfd_create_supported) {
const int in_memory_flags = MAP_HUGETLB | MAP_FIXED |
MAP_PRIVATE | MAP_ANONYMOUS;
int pagesz_flag;
return -1;
}
- if (internal_config.single_file_segments) {
+ if (internal_conf->single_file_segments) {
map_offset = seg_idx * alloc_sz;
ret = resize_hugefile(fd, map_offset, alloc_sz, true);
if (ret < 0)
__func__, strerror(errno));
goto resized;
}
- if (internal_config.hugepage_unlink &&
- !internal_config.in_memory) {
+ if (internal_conf->hugepage_unlink &&
+ !internal_conf->in_memory) {
if (unlink(path)) {
RTE_LOG(DEBUG, EAL, "%s(): unlink() failed: %s\n",
__func__, strerror(errno));
}
#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
- ret = get_mempolicy(&cur_socket_id, NULL, 0, addr,
- MPOL_F_NODE | MPOL_F_ADDR);
- if (ret < 0) {
- RTE_LOG(DEBUG, EAL, "%s(): get_mempolicy: %s\n",
- __func__, strerror(errno));
- goto mapped;
- } else if (cur_socket_id != socket_id) {
- RTE_LOG(DEBUG, EAL,
- "%s(): allocation happened on wrong socket (wanted %d, got %d)\n",
- __func__, socket_id, cur_socket_id);
- goto mapped;
+ /*
+ * If the kernel has been built without NUMA support, get_mempolicy()
+ * will return an error. If check_numa() returns false, memory
+ * allocation is not NUMA aware and the socket_id should not be
+ * checked.
+ */
+ if (check_numa()) {
+ ret = get_mempolicy(&cur_socket_id, NULL, 0, addr,
+ MPOL_F_NODE | MPOL_F_ADDR);
+ if (ret < 0) {
+ RTE_LOG(DEBUG, EAL, "%s(): get_mempolicy: %s\n",
+ __func__, strerror(errno));
+ goto mapped;
+ } else if (cur_socket_id != socket_id) {
+ RTE_LOG(DEBUG, EAL,
+ "%s(): allocation happened on wrong socket (wanted %d, got %d)\n",
+ __func__, socket_id, cur_socket_id);
+ goto mapped;
+ }
}
#else
if (rte_socket_count() > 1)
mapped:
munmap(addr, alloc_sz);
unmapped:
- flags = MAP_FIXED;
+ flags = EAL_RESERVE_FORCE_ADDRESS;
new_addr = eal_get_virtual_area(addr, &alloc_sz, alloc_sz, 0, flags);
if (new_addr != addr) {
if (new_addr != NULL)
RTE_LOG(CRIT, EAL, "Can't mmap holes in our virtual address space\n");
}
/* roll back the ref count */
- if (internal_config.single_file_segments)
+ if (internal_conf->single_file_segments)
fd_list[list_idx].count--;
resized:
/* some codepaths will return negative fd, so exit early */
if (fd < 0)
return -1;
- if (internal_config.single_file_segments) {
+ if (internal_conf->single_file_segments) {
resize_hugefile(fd, map_offset, alloc_sz, false);
/* ignore failure, can't make it any worse */
close_hugefile(fd, path, list_idx);
} else {
/* only remove file if we can take out a write lock */
- if (internal_config.hugepage_unlink == 0 &&
- internal_config.in_memory == 0 &&
+ if (internal_conf->hugepage_unlink == 0 &&
+ internal_conf->in_memory == 0 &&
lock(fd, LOCK_EX) == 1)
unlink(path);
close(fd);
char path[PATH_MAX];
int fd, ret = 0;
bool exit_early;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
/* erase page data */
memset(ms->addr, 0, ms->len);
return -1;
}
+ eal_mem_set_dump(ms->addr, ms->len, false);
+
exit_early = false;
/* if we're using anonymous hugepages, nothing to be done */
- if (internal_config.in_memory && !memfd_create_supported)
+ if (internal_conf->in_memory && !memfd_create_supported)
exit_early = true;
/* if we've already unlinked the page, nothing needs to be done */
- if (!internal_config.in_memory && internal_config.hugepage_unlink)
+ if (!internal_conf->in_memory && internal_conf->hugepage_unlink)
exit_early = true;
if (exit_early) {
if (fd < 0)
return -1;
- if (internal_config.single_file_segments) {
+ if (internal_conf->single_file_segments) {
map_offset = seg_idx * ms->len;
if (resize_hugefile(fd, map_offset, ms->len, false))
return -1;
/* if we're able to take out a write lock, we're the last one
* holding onto this page.
*/
- if (!internal_config.in_memory) {
+ if (!internal_conf->in_memory) {
ret = lock(fd, LOCK_EX);
if (ret >= 0) {
/* no one else is using this page */
size_t page_sz;
int cur_idx, start_idx, j, dir_fd = -1;
unsigned int msl_idx, need, i;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
if (msl->page_sz != wa->page_sz)
return 0;
* during init, we already hold a write lock, so don't try to take out
* another one.
*/
- if (wa->hi->lock_descriptor == -1 && !internal_config.in_memory) {
+ if (wa->hi->lock_descriptor == -1 && !internal_conf->in_memory) {
dir_fd = open(wa->hi->hugedir, O_RDONLY);
if (dir_fd < 0) {
RTE_LOG(ERR, EAL, "%s(): Cannot open '%s': %s\n",
struct free_walk_param *wa = arg;
uintptr_t start_addr, end_addr;
int msl_idx, seg_idx, ret, dir_fd = -1;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
start_addr = (uintptr_t) msl->base_va;
end_addr = start_addr + msl->len;
* during init, we already hold a write lock, so don't try to take out
* another one.
*/
- if (wa->hi->lock_descriptor == -1 && !internal_config.in_memory) {
+ if (wa->hi->lock_descriptor == -1 && !internal_conf->in_memory) {
dir_fd = open(wa->hi->hugedir, O_RDONLY);
if (dir_fd < 0) {
RTE_LOG(ERR, EAL, "%s(): Cannot open '%s': %s\n",
#endif
struct alloc_walk_param wa;
struct hugepage_info *hi = NULL;
+ struct internal_config *internal_conf =
+ eal_get_internal_configuration();
memset(&wa, 0, sizeof(wa));
/* dynamic allocation not supported in legacy mode */
- if (internal_config.legacy_mem)
+ if (internal_conf->legacy_mem)
return -1;
- for (i = 0; i < (int) RTE_DIM(internal_config.hugepage_info); i++) {
+ for (i = 0; i < (int) RTE_DIM(internal_conf->hugepage_info); i++) {
if (page_sz ==
- internal_config.hugepage_info[i].hugepage_sz) {
- hi = &internal_config.hugepage_info[i];
+ internal_conf->hugepage_info[i].hugepage_sz) {
+ hi = &internal_conf->hugepage_info[i];
break;
}
}
eal_memalloc_free_seg_bulk(struct rte_memseg **ms, int n_segs)
{
int seg, ret = 0;
+ struct internal_config *internal_conf =
+ eal_get_internal_configuration();
/* dynamic free not supported in legacy mode */
- if (internal_config.legacy_mem)
+ if (internal_conf->legacy_mem)
return -1;
for (seg = 0; seg < n_segs; seg++) {
memset(&wa, 0, sizeof(wa));
- for (i = 0; i < (int)RTE_DIM(internal_config.hugepage_info);
+ for (i = 0; i < (int)RTE_DIM(internal_conf->hugepage_info);
i++) {
- hi = &internal_config.hugepage_info[i];
+ hi = &internal_conf->hugepage_info[i];
if (cur->hugepage_sz == hi->hugepage_sz)
break;
}
- if (i == (int)RTE_DIM(internal_config.hugepage_info)) {
+ if (i == (int)RTE_DIM(internal_conf->hugepage_info)) {
RTE_LOG(ERR, EAL, "Can't find relevant hugepage_info entry\n");
ret = -1;
continue;
int
eal_memalloc_free_seg(struct rte_memseg *ms)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
/* dynamic free not supported in legacy mode */
- if (internal_config.legacy_mem)
+ if (internal_conf->legacy_mem)
return -1;
return eal_memalloc_free_seg_bulk(&ms, 1);
struct hugepage_info *hi = NULL;
unsigned int i;
int msl_idx;
+ struct internal_config *internal_conf =
+ eal_get_internal_configuration();
if (msl->external)
return 0;
primary_msl = &mcfg->memsegs[msl_idx];
local_msl = &local_memsegs[msl_idx];
- for (i = 0; i < RTE_DIM(internal_config.hugepage_info); i++) {
+ for (i = 0; i < RTE_DIM(internal_conf->hugepage_info); i++) {
uint64_t cur_sz =
- internal_config.hugepage_info[i].hugepage_sz;
+ internal_conf->hugepage_info[i].hugepage_sz;
uint64_t msl_sz = primary_msl->page_sz;
if (msl_sz == cur_sz) {
- hi = &internal_config.hugepage_info[i];
+ hi = &internal_conf->hugepage_info[i];
break;
}
}
{
int *data;
int i;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
/* single-file segments mode does not need fd list */
- if (!internal_config.single_file_segments) {
+ if (!internal_conf->single_file_segments) {
/* ensure we have space to store fd per each possible segment */
data = malloc(sizeof(int) * len);
if (data == NULL) {
eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
/* single file segments mode doesn't support individual segment fd's */
- if (internal_config.single_file_segments)
+ if (internal_conf->single_file_segments)
return -ENOTSUP;
/* if list is not allocated, allocate it */
int
eal_memalloc_set_seg_list_fd(int list_idx, int fd)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
/* non-single file segment mode doesn't support segment list fd's */
- if (!internal_config.single_file_segments)
+ if (!internal_conf->single_file_segments)
return -ENOTSUP;
fd_list[list_idx].memseg_list_fd = fd;
eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
{
int fd;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
- if (internal_config.in_memory || internal_config.no_hugetlbfs) {
+ if (internal_conf->in_memory || internal_conf->no_hugetlbfs) {
#ifndef MEMFD_SUPPORTED
/* in in-memory or no-huge mode, we rely on memfd support */
return -ENOTSUP;
#endif
/* memfd supported, but hugetlbfs memfd may not be */
- if (!internal_config.no_hugetlbfs && !memfd_create_supported)
+ if (!internal_conf->no_hugetlbfs && !memfd_create_supported)
return -ENOTSUP;
}
- if (internal_config.single_file_segments) {
+ if (internal_conf->single_file_segments) {
fd = fd_list[list_idx].memseg_list_fd;
} else if (fd_list[list_idx].len == 0) {
/* list not initialized */
test_memfd_create(void)
{
#ifdef MEMFD_SUPPORTED
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
unsigned int i;
- for (i = 0; i < internal_config.num_hugepage_sizes; i++) {
- uint64_t pagesz = internal_config.hugepage_info[i].hugepage_sz;
+ for (i = 0; i < internal_conf->num_hugepage_sizes; i++) {
+ uint64_t pagesz = internal_conf->hugepage_info[i].hugepage_sz;
int pagesz_flag = pagesz_flags(pagesz);
int flags;
eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
- if (internal_config.in_memory || internal_config.no_hugetlbfs) {
+ if (internal_conf->in_memory || internal_conf->no_hugetlbfs) {
#ifndef MEMFD_SUPPORTED
/* in in-memory or no-huge mode, we rely on memfd support */
return -ENOTSUP;
#endif
/* memfd supported, but hugetlbfs memfd may not be */
- if (!internal_config.no_hugetlbfs && !memfd_create_supported)
+ if (!internal_conf->no_hugetlbfs && !memfd_create_supported)
return -ENOTSUP;
}
- if (internal_config.single_file_segments) {
+ if (internal_conf->single_file_segments) {
size_t pgsz = mcfg->memsegs[list_idx].page_sz;
/* segment not active? */
int
eal_memalloc_init(void)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
if (rte_eal_process_type() == RTE_PROC_SECONDARY)
if (rte_memseg_list_walk(secondary_msl_create_walk, NULL) < 0)
return -1;
if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
- internal_config.in_memory) {
+ internal_conf->in_memory) {
int mfd_res = test_memfd_create();
if (mfd_res < 0) {
* if we support hugetlbfs with memfd_create. this code will
* test if we do.
*/
- if (internal_config.single_file_segments &&
+ if (internal_conf->single_file_segments &&
mfd_res != 1) {
RTE_LOG(ERR, EAL, "Single-file segments mode cannot be used without memfd support\n");
return -1;
git.droids-corp.org / dpdk.git / blobdiff