1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2020 Dmitry Kozlyuk
7 #include "eal_internal_cfg.h"
8 #include "eal_memalloc.h"
9 #include "eal_memcfg.h"
10 #include "eal_private.h"
11 #include "eal_windows.h"
14 eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
16 /* Hugepages have no associated files in Windows. */
17 RTE_SET_USED(list_idx);
18 RTE_SET_USED(seg_idx);
19 EAL_LOG_NOT_IMPLEMENTED();
24 eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset)
26 /* Hugepages have no associated files in Windows. */
27 RTE_SET_USED(list_idx);
28 RTE_SET_USED(seg_idx);
30 EAL_LOG_NOT_IMPLEMENTED();
35 alloc_seg(struct rte_memseg *ms, void *requested_addr, int socket_id,
36 struct hugepage_info *hi)
38 HANDLE current_process;
39 unsigned int numa_node;
42 rte_iova_t iova = RTE_BAD_IOVA;
43 PSAPI_WORKING_SET_EX_INFORMATION info;
44 PSAPI_WORKING_SET_EX_BLOCK *page;
47 /* If a segment is already allocated as needed, return it. */
48 if ((ms->addr == requested_addr) &&
49 (ms->socket_id == socket_id) &&
50 (ms->hugepage_sz == hi->hugepage_sz)) {
54 /* Bugcheck, should not happen. */
55 RTE_LOG(DEBUG, EAL, "Attempted to reallocate segment %p "
56 "(size %zu) on socket %d", ms->addr,
57 ms->len, ms->socket_id);
61 current_process = GetCurrentProcess();
62 numa_node = eal_socket_numa_node(socket_id);
63 alloc_sz = hi->hugepage_sz;
65 if (requested_addr == NULL) {
66 /* Request a new chunk of memory from OS. */
67 addr = eal_mem_alloc_socket(alloc_sz, socket_id);
69 RTE_LOG(DEBUG, EAL, "Cannot allocate %zu bytes "
70 "on socket %d\n", alloc_sz, socket_id);
74 /* Requested address is already reserved, commit memory. */
75 addr = eal_mem_commit(requested_addr, alloc_sz, socket_id);
77 /* During commitment, memory is temporary freed and might
78 * be allocated by different non-EAL thread. This is a fatal
79 * error, because it breaks MSL assumptions.
81 if ((addr != NULL) && (addr != requested_addr)) {
82 RTE_LOG(CRIT, EAL, "Address %p occupied by an alien "
83 " allocation - MSL is not VA-contiguous!\n",
89 RTE_LOG(DEBUG, EAL, "Cannot commit reserved memory %p "
90 "(size %zu) on socket %d\n",
91 requested_addr, alloc_sz, socket_id);
96 /* Force OS to allocate a physical page and select a NUMA node.
97 * Hugepages are not pageable in Windows, so there's no race
98 * for physical address.
100 *(volatile int *)addr = *(volatile int *)addr;
102 iova = rte_mem_virt2iova(addr);
103 if (iova == RTE_BAD_IOVA) {
105 "Cannot get IOVA of allocated segment\n");
109 /* Only "Ex" function can handle hugepages. */
110 info.VirtualAddress = addr;
111 if (!QueryWorkingSetEx(current_process, &info, sizeof(info))) {
112 RTE_LOG_WIN32_ERR("QueryWorkingSetEx(%p)", addr);
116 page = &info.VirtualAttributes;
117 if (!page->Valid || !page->LargePage) {
118 RTE_LOG(DEBUG, EAL, "Got regular page instead of a hugepage\n");
121 if (page->Node != numa_node) {
123 "NUMA node hint %u (socket %d) not respected, got %u\n",
124 numa_node, socket_id, page->Node);
129 ms->hugepage_sz = hi->hugepage_sz;
131 ms->nchannel = rte_memory_get_nchannel();
132 ms->nrank = rte_memory_get_nrank();
134 ms->socket_id = socket_id;
139 /* Only jump here when `addr` and `alloc_sz` are valid. */
140 if (eal_mem_decommit(addr, alloc_sz) && (rte_errno == EADDRNOTAVAIL)) {
141 /* During decommitment, memory is temporarily returned
142 * to the system and the address may become unavailable.
144 RTE_LOG(CRIT, EAL, "Address %p occupied by an alien "
145 " allocation - MSL is not VA-contiguous!\n", addr);
151 free_seg(struct rte_memseg *ms)
153 if (eal_mem_decommit(ms->addr, ms->len)) {
154 if (rte_errno == EADDRNOTAVAIL) {
155 /* See alloc_seg() for explanation. */
156 RTE_LOG(CRIT, EAL, "Address %p occupied by an alien "
157 " allocation - MSL is not VA-contiguous!\n",
163 /* Must clear the segment, because alloc_seg() inspects it. */
164 memset(ms, 0, sizeof(*ms));
168 struct alloc_walk_param {
169 struct hugepage_info *hi;
170 struct rte_memseg **ms;
172 unsigned int segs_allocated;
179 alloc_seg_walk(const struct rte_memseg_list *msl, void *arg)
181 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
182 struct alloc_walk_param *wa = arg;
183 struct rte_memseg_list *cur_msl;
185 int cur_idx, start_idx, j;
186 unsigned int msl_idx, need, i;
188 if (msl->page_sz != wa->page_sz)
190 if (msl->socket_id != wa->socket)
193 page_sz = (size_t)msl->page_sz;
195 msl_idx = msl - mcfg->memsegs;
196 cur_msl = &mcfg->memsegs[msl_idx];
200 /* try finding space in memseg list */
202 /* if we require exact number of pages in a list, find them */
203 cur_idx = rte_fbarray_find_next_n_free(
204 &cur_msl->memseg_arr, 0, need);
211 /* we don't require exact number of pages, so we're going to go
212 * for best-effort allocation. that means finding the biggest
213 * unused block, and going with that.
215 cur_idx = rte_fbarray_find_biggest_free(
216 &cur_msl->memseg_arr, 0);
220 /* adjust the size to possibly be smaller than original
221 * request, but do not allow it to be bigger.
223 cur_len = rte_fbarray_find_contig_free(
224 &cur_msl->memseg_arr, cur_idx);
225 need = RTE_MIN(need, (unsigned int)cur_len);
228 for (i = 0; i < need; i++, cur_idx++) {
229 struct rte_memseg *cur;
232 cur = rte_fbarray_get(&cur_msl->memseg_arr, cur_idx);
233 map_addr = RTE_PTR_ADD(cur_msl->base_va, cur_idx * page_sz);
235 if (alloc_seg(cur, map_addr, wa->socket, wa->hi)) {
236 RTE_LOG(DEBUG, EAL, "attempted to allocate %i segments, "
237 "but only %i were allocated\n", need, i);
239 /* if exact number wasn't requested, stop */
244 for (j = start_idx; j < cur_idx; j++) {
245 struct rte_memseg *tmp;
246 struct rte_fbarray *arr = &cur_msl->memseg_arr;
248 tmp = rte_fbarray_get(arr, j);
249 rte_fbarray_set_free(arr, j);
252 RTE_LOG(DEBUG, EAL, "Cannot free page\n");
256 memset(wa->ms, 0, sizeof(*wa->ms) * wa->n_segs);
263 rte_fbarray_set_used(&cur_msl->memseg_arr, cur_idx);
267 wa->segs_allocated = i;
271 /* if we didn't allocate any segments, move on to the next list */
275 struct free_walk_param {
276 struct hugepage_info *hi;
277 struct rte_memseg *ms;
280 free_seg_walk(const struct rte_memseg_list *msl, void *arg)
282 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
283 struct rte_memseg_list *found_msl;
284 struct free_walk_param *wa = arg;
285 uintptr_t start_addr, end_addr;
286 int msl_idx, seg_idx, ret;
288 start_addr = (uintptr_t) msl->base_va;
289 end_addr = start_addr + msl->len;
291 if ((uintptr_t)wa->ms->addr < start_addr ||
292 (uintptr_t)wa->ms->addr >= end_addr)
295 msl_idx = msl - mcfg->memsegs;
296 seg_idx = RTE_PTR_DIFF(wa->ms->addr, start_addr) / msl->page_sz;
299 found_msl = &mcfg->memsegs[msl_idx];
300 found_msl->version++;
302 rte_fbarray_set_free(&found_msl->memseg_arr, seg_idx);
304 ret = free_seg(wa->ms);
306 return (ret < 0) ? (-1) : 1;
310 eal_memalloc_alloc_seg_bulk(struct rte_memseg **ms, int n_segs,
311 size_t page_sz, int socket, bool exact)
315 struct alloc_walk_param wa;
316 struct hugepage_info *hi = NULL;
317 struct internal_config *internal_conf =
318 eal_get_internal_configuration();
320 if (internal_conf->legacy_mem) {
321 RTE_LOG(ERR, EAL, "dynamic allocation not supported in legacy mode\n");
325 for (i = 0; i < internal_conf->num_hugepage_sizes; i++) {
326 struct hugepage_info *hpi = &internal_conf->hugepage_info[i];
327 if (page_sz == hpi->hugepage_sz) {
333 RTE_LOG(ERR, EAL, "cannot find relevant hugepage_info entry\n");
337 memset(&wa, 0, sizeof(wa));
342 wa.page_sz = page_sz;
344 wa.segs_allocated = 0;
346 /* memalloc is locked, so it's safe to use thread-unsafe version */
347 ret = rte_memseg_list_walk_thread_unsafe(alloc_seg_walk, &wa);
349 RTE_LOG(ERR, EAL, "cannot find suitable memseg_list\n");
351 } else if (ret > 0) {
352 ret = (int)wa.segs_allocated;
359 eal_memalloc_alloc_seg(size_t page_sz, int socket)
361 struct rte_memseg *ms = NULL;
362 eal_memalloc_alloc_seg_bulk(&ms, 1, page_sz, socket, true);
367 eal_memalloc_free_seg_bulk(struct rte_memseg **ms, int n_segs)
370 struct internal_config *internal_conf =
371 eal_get_internal_configuration();
373 /* dynamic free not supported in legacy mode */
374 if (internal_conf->legacy_mem)
377 for (seg = 0; seg < n_segs; seg++) {
378 struct rte_memseg *cur = ms[seg];
379 struct hugepage_info *hi = NULL;
380 struct free_walk_param wa;
384 /* if this page is marked as unfreeable, fail */
385 if (cur->flags & RTE_MEMSEG_FLAG_DO_NOT_FREE) {
386 RTE_LOG(DEBUG, EAL, "Page is not allowed to be freed\n");
391 memset(&wa, 0, sizeof(wa));
393 for (i = 0; i < RTE_DIM(internal_conf->hugepage_info); i++) {
394 hi = &internal_conf->hugepage_info[i];
395 if (cur->hugepage_sz == hi->hugepage_sz)
398 if (i == RTE_DIM(internal_conf->hugepage_info)) {
399 RTE_LOG(ERR, EAL, "Can't find relevant hugepage_info entry\n");
407 /* memalloc is locked, so it's safe to use thread-unsafe version
409 walk_res = rte_memseg_list_walk_thread_unsafe(free_seg_walk,
414 RTE_LOG(ERR, EAL, "Couldn't find memseg list\n");
421 eal_memalloc_free_seg(struct rte_memseg *ms)
423 return eal_memalloc_free_seg_bulk(&ms, 1);
427 eal_memalloc_sync_with_primary(void)
429 /* No multi-process support. */
430 EAL_LOG_NOT_IMPLEMENTED();
435 eal_memalloc_cleanup(void)
437 /* not implemented */
442 eal_memalloc_init(void)
444 /* No action required. */