1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
6 #include <rte_malloc.h>
11 #include "mlx5_common_log.h"
12 #include "mlx5_common_os.h"
13 #include "mlx5_malloc.h"
16 uint32_t init:1; /* Memory allocator initialized. */
17 uint32_t enable:1; /* System memory select. */
18 uint32_t reserve:30; /* Reserve. */
19 struct rte_memseg_list *last_msl;
20 /* last allocated rte memory memseg list. */
21 #ifdef RTE_LIBRTE_MLX5_DEBUG
23 /* Memory allocated from system count. */
25 /* Memory allocated from hugepage count. */
27 /* Memory reallocate from system count. */
29 /* Memory reallocate from hugepage count. */
31 /* Memory free to system count. */
33 /* Memory free to hugepage count. */
37 /* MSL update count. */
41 /* Initialize default as not */
42 static struct mlx5_sys_mem mlx5_sys_mem = {
45 #ifdef RTE_LIBRTE_MLX5_DEBUG
58 * Check if the address belongs to memory seg list.
61 * Memory address to be checked.
66 * True if it belongs, false otherwise.
69 mlx5_mem_check_msl(void *addr, struct rte_memseg_list *msl)
76 end = RTE_PTR_ADD(start, msl->len);
77 if (addr >= start && addr < end)
83 * Update the msl if memory belongs to new msl.
89 mlx5_mem_update_msl(void *addr)
92 * Update the cache msl if the new addr comes from the new msl
93 * different with the cached msl.
95 if (addr && !mlx5_mem_check_msl(addr,
96 (struct rte_memseg_list *)__atomic_load_n
97 (&mlx5_sys_mem.last_msl, __ATOMIC_RELAXED))) {
98 __atomic_store_n(&mlx5_sys_mem.last_msl,
99 rte_mem_virt2memseg_list(addr),
101 #ifdef RTE_LIBRTE_MLX5_DEBUG
102 __atomic_add_fetch(&mlx5_sys_mem.msl_update, 1,
109 * Check if the address belongs to rte memory.
112 * Memory address to be checked.
115 * True if it belongs, false otherwise.
118 mlx5_mem_is_rte(void *addr)
121 * Check if the last cache msl matches. Drop to slow path
122 * to check if the memory belongs to rte memory.
124 if (!mlx5_mem_check_msl(addr, (struct rte_memseg_list *)
125 __atomic_load_n(&mlx5_sys_mem.last_msl, __ATOMIC_RELAXED))) {
126 if (!rte_mem_virt2memseg_list(addr))
128 #ifdef RTE_LIBRTE_MLX5_DEBUG
129 __atomic_add_fetch(&mlx5_sys_mem.msl_miss, 1, __ATOMIC_RELAXED);
136 * Allocate memory with alignment.
139 * Memory size to be allocated.
143 * Clear the allocated memory or not.
146 * Pointer of the allocated memory, NULL otherwise.
149 mlx5_alloc_align(size_t size, unsigned int align, unsigned int zero)
153 buf = mlx5_os_malloc(align, size);
155 DRV_LOG(ERR, "Couldn't allocate buf size=%zu align=%u.",
160 memset(buf, 0, size);
165 mlx5_malloc(uint32_t flags, size_t size, unsigned int align, int socket)
171 * If neither system memory nor rte memory is required, allocate
172 * memory according to mlx5_sys_mem.enable.
174 if (flags & MLX5_MEM_RTE)
176 else if (flags & MLX5_MEM_SYS)
179 rte_mem = mlx5_sys_mem.enable ? false : true;
181 if (flags & MLX5_MEM_ZERO)
182 addr = rte_zmalloc_socket(NULL, size, align, socket);
184 addr = rte_malloc_socket(NULL, size, align, socket);
185 mlx5_mem_update_msl(addr);
186 #ifdef RTE_LIBRTE_MLX5_DEBUG
188 __atomic_add_fetch(&mlx5_sys_mem.malloc_rte, 1,
193 /* The memory will be allocated from system. */
194 if (align > MLX5_MALLOC_ALIGNMENT)
195 addr = mlx5_alloc_align(size, align, !!(flags & MLX5_MEM_ZERO));
196 else if (flags & MLX5_MEM_ZERO)
197 addr = calloc(1, size);
200 #ifdef RTE_LIBRTE_MLX5_DEBUG
202 __atomic_add_fetch(&mlx5_sys_mem.malloc_sys, 1,
209 mlx5_realloc(void *addr, uint32_t flags, size_t size, unsigned int align,
215 /* Allocate directly if old memory address is NULL. */
217 return mlx5_malloc(flags, size, align, socket);
218 /* Get the memory type. */
219 if (flags & MLX5_MEM_RTE)
221 else if (flags & MLX5_MEM_SYS)
224 rte_mem = mlx5_sys_mem.enable ? false : true;
225 /* Check if old memory and to be allocated memory are the same type. */
226 if (rte_mem != mlx5_mem_is_rte(addr)) {
227 DRV_LOG(ERR, "Couldn't reallocate to different memory type.");
230 /* Allocate memory from rte memory. */
232 new_addr = rte_realloc_socket(addr, size, align, socket);
233 mlx5_mem_update_msl(new_addr);
234 #ifdef RTE_LIBRTE_MLX5_DEBUG
236 __atomic_add_fetch(&mlx5_sys_mem.realloc_rte, 1,
241 /* Align is not supported for system memory. */
243 DRV_LOG(ERR, "Couldn't reallocate with alignment");
246 new_addr = realloc(addr, size);
247 #ifdef RTE_LIBRTE_MLX5_DEBUG
249 __atomic_add_fetch(&mlx5_sys_mem.realloc_sys, 1,
256 mlx5_free(void *addr)
260 if (!mlx5_mem_is_rte(addr)) {
261 #ifdef RTE_LIBRTE_MLX5_DEBUG
262 __atomic_add_fetch(&mlx5_sys_mem.free_sys, 1,
267 #ifdef RTE_LIBRTE_MLX5_DEBUG
268 __atomic_add_fetch(&mlx5_sys_mem.free_rte, 1,
276 mlx5_memory_stat_dump(void)
278 #ifdef RTE_LIBRTE_MLX5_DEBUG
279 DRV_LOG(INFO, "System memory malloc:%"PRIi64", realloc:%"PRIi64","
280 " free:%"PRIi64"\nRTE memory malloc:%"PRIi64","
281 " realloc:%"PRIi64", free:%"PRIi64"\nMSL miss:%"PRIi64","
283 __atomic_load_n(&mlx5_sys_mem.malloc_sys, __ATOMIC_RELAXED),
284 __atomic_load_n(&mlx5_sys_mem.realloc_sys, __ATOMIC_RELAXED),
285 __atomic_load_n(&mlx5_sys_mem.free_sys, __ATOMIC_RELAXED),
286 __atomic_load_n(&mlx5_sys_mem.malloc_rte, __ATOMIC_RELAXED),
287 __atomic_load_n(&mlx5_sys_mem.realloc_rte, __ATOMIC_RELAXED),
288 __atomic_load_n(&mlx5_sys_mem.free_rte, __ATOMIC_RELAXED),
289 __atomic_load_n(&mlx5_sys_mem.msl_miss, __ATOMIC_RELAXED),
290 __atomic_load_n(&mlx5_sys_mem.msl_update, __ATOMIC_RELAXED));
295 mlx5_malloc_mem_select(uint32_t sys_mem_en)
298 * The initialization should be called only once and all devices
299 * should use the same memory type. Otherwise, when new device is
300 * being attached with some different memory allocation configuration,
301 * the memory will get wrong behavior or a failure will be raised.
303 if (!mlx5_sys_mem.init) {
305 mlx5_sys_mem.enable = 1;
306 mlx5_sys_mem.init = 1;
307 DRV_LOG(INFO, "%s is selected.", sys_mem_en ? "SYS_MEM" : "RTE_MEM");
308 } else if (mlx5_sys_mem.enable != sys_mem_en) {
309 DRV_LOG(WARNING, "%s is already selected.",
310 mlx5_sys_mem.enable ? "SYS_MEM" : "RTE_MEM");