1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
6 #include <rte_malloc.h>
11 #include <rte_atomic.h>
13 #include "mlx5_common_utils.h"
14 #include "mlx5_malloc.h"
17 uint32_t init:1; /* Memory allocator initialized. */
18 uint32_t enable:1; /* System memory select. */
19 uint32_t reserve:30; /* Reserve. */
21 struct rte_memseg_list *last_msl;
22 rte_atomic64_t a64_last_msl;
24 /* last allocated rte memory memseg list. */
25 #ifdef RTE_LIBRTE_MLX5_DEBUG
26 rte_atomic64_t malloc_sys;
27 /* Memory allocated from system count. */
28 rte_atomic64_t malloc_rte;
29 /* Memory allocated from hugepage count. */
30 rte_atomic64_t realloc_sys;
31 /* Memory reallocate from system count. */
32 rte_atomic64_t realloc_rte;
33 /* Memory reallocate from hugepage count. */
34 rte_atomic64_t free_sys;
35 /* Memory free to system count. */
36 rte_atomic64_t free_rte;
37 /* Memory free to hugepage count. */
38 rte_atomic64_t msl_miss;
40 rte_atomic64_t msl_update;
41 /* MSL update count. */
45 /* Initialize default as not */
46 static struct mlx5_sys_mem mlx5_sys_mem = {
49 #ifdef RTE_LIBRTE_MLX5_DEBUG
50 .malloc_sys = RTE_ATOMIC64_INIT(0),
51 .malloc_rte = RTE_ATOMIC64_INIT(0),
52 .realloc_sys = RTE_ATOMIC64_INIT(0),
53 .realloc_rte = RTE_ATOMIC64_INIT(0),
54 .free_sys = RTE_ATOMIC64_INIT(0),
55 .free_rte = RTE_ATOMIC64_INIT(0),
56 .msl_miss = RTE_ATOMIC64_INIT(0),
57 .msl_update = RTE_ATOMIC64_INIT(0),
62 * Check if the address belongs to memory seg list.
65 * Memory address to be ckeced.
70 * True if it belongs, false otherwise.
73 mlx5_mem_check_msl(void *addr, struct rte_memseg_list *msl)
80 end = RTE_PTR_ADD(start, msl->len);
81 if (addr >= start && addr < end)
87 * Update the msl if memory belongs to new msl.
93 mlx5_mem_update_msl(void *addr)
96 * Update the cache msl if the new addr comes from the new msl
97 * different with the cached msl.
99 if (addr && !mlx5_mem_check_msl(addr,
100 (struct rte_memseg_list *)(uintptr_t)rte_atomic64_read
101 (&mlx5_sys_mem.a64_last_msl))) {
102 rte_atomic64_set(&mlx5_sys_mem.a64_last_msl,
103 (int64_t)(uintptr_t)rte_mem_virt2memseg_list(addr));
104 #ifdef RTE_LIBRTE_MLX5_DEBUG
105 rte_atomic64_inc(&mlx5_sys_mem.msl_update);
111 * Check if the address belongs to rte memory.
114 * Memory address to be ckeced.
117 * True if it belongs, false otherwise.
120 mlx5_mem_is_rte(void *addr)
123 * Check if the last cache msl matches. Drop to slow path
124 * to check if the memory belongs to rte memory.
126 if (!mlx5_mem_check_msl(addr, (struct rte_memseg_list *)(uintptr_t)
127 rte_atomic64_read(&mlx5_sys_mem.a64_last_msl))) {
128 if (!rte_mem_virt2memseg_list(addr))
130 #ifdef RTE_LIBRTE_MLX5_DEBUG
131 rte_atomic64_inc(&mlx5_sys_mem.msl_miss);
138 * Allocate memory with alignment.
141 * Memory size to be allocated.
145 * Clear the allocated memory or not.
148 * Pointer of the allocated memory, NULL otherwise.
151 mlx5_alloc_align(size_t size, unsigned int align, unsigned int zero)
156 ret = posix_memalign(&buf, align, size);
159 "Couldn't allocate buf size=%zu align=%u. Err=%d\n",
165 memset(buf, 0, size);
170 mlx5_malloc(uint32_t flags, size_t size, unsigned int align, int socket)
176 * If neither system memory nor rte memory is required, allocate
177 * memory according to mlx5_sys_mem.enable.
179 if (flags & MLX5_MEM_RTE)
181 else if (flags & MLX5_MEM_SYS)
184 rte_mem = mlx5_sys_mem.enable ? false : true;
186 if (flags & MLX5_MEM_ZERO)
187 addr = rte_zmalloc_socket(NULL, size, align, socket);
189 addr = rte_malloc_socket(NULL, size, align, socket);
190 mlx5_mem_update_msl(addr);
191 #ifdef RTE_LIBRTE_MLX5_DEBUG
193 rte_atomic64_inc(&mlx5_sys_mem.malloc_rte);
197 /* The memory will be allocated from system. */
198 if (align > MLX5_MALLOC_ALIGNMENT)
199 addr = mlx5_alloc_align(size, align, !!(flags & MLX5_MEM_ZERO));
200 else if (flags & MLX5_MEM_ZERO)
201 addr = calloc(1, size);
204 #ifdef RTE_LIBRTE_MLX5_DEBUG
206 rte_atomic64_inc(&mlx5_sys_mem.malloc_sys);
212 mlx5_realloc(void *addr, uint32_t flags, size_t size, unsigned int align,
218 /* Allocate directly if old memory address is NULL. */
220 return mlx5_malloc(flags, size, align, socket);
221 /* Get the memory type. */
222 if (flags & MLX5_MEM_RTE)
224 else if (flags & MLX5_MEM_SYS)
227 rte_mem = mlx5_sys_mem.enable ? false : true;
228 /* Check if old memory and to be allocated memory are the same type. */
229 if (rte_mem != mlx5_mem_is_rte(addr)) {
230 DRV_LOG(ERR, "Couldn't reallocate to different memory type.");
233 /* Allocate memory from rte memory. */
235 new_addr = rte_realloc_socket(addr, size, align, socket);
236 mlx5_mem_update_msl(new_addr);
237 #ifdef RTE_LIBRTE_MLX5_DEBUG
239 rte_atomic64_inc(&mlx5_sys_mem.realloc_rte);
243 /* Align is not supported for system memory. */
245 DRV_LOG(ERR, "Couldn't reallocate with alignment");
248 new_addr = realloc(addr, size);
249 #ifdef RTE_LIBRTE_MLX5_DEBUG
251 rte_atomic64_inc(&mlx5_sys_mem.realloc_sys);
257 mlx5_free(void *addr)
261 if (!mlx5_mem_is_rte(addr)) {
262 #ifdef RTE_LIBRTE_MLX5_DEBUG
263 rte_atomic64_inc(&mlx5_sys_mem.free_sys);
267 #ifdef RTE_LIBRTE_MLX5_DEBUG
268 rte_atomic64_inc(&mlx5_sys_mem.free_rte);
275 mlx5_memory_stat_dump(void)
277 #ifdef RTE_LIBRTE_MLX5_DEBUG
278 DRV_LOG(INFO, "System memory malloc:%"PRIi64", realloc:%"PRIi64","
279 " free:%"PRIi64"\nRTE memory malloc:%"PRIi64","
280 " realloc:%"PRIi64", free:%"PRIi64"\nMSL miss:%"PRIi64","
282 rte_atomic64_read(&mlx5_sys_mem.malloc_sys),
283 rte_atomic64_read(&mlx5_sys_mem.realloc_sys),
284 rte_atomic64_read(&mlx5_sys_mem.free_sys),
285 rte_atomic64_read(&mlx5_sys_mem.malloc_rte),
286 rte_atomic64_read(&mlx5_sys_mem.realloc_rte),
287 rte_atomic64_read(&mlx5_sys_mem.free_rte),
288 rte_atomic64_read(&mlx5_sys_mem.msl_miss),
289 rte_atomic64_read(&mlx5_sys_mem.msl_update));
294 mlx5_malloc_mem_select(uint32_t sys_mem_en)
297 * The initialization should be called only once and all devices
298 * should use the same memory type. Otherwise, when new device is
299 * being attached with some different memory allocation configuration,
300 * the memory will get wrong behavior or a failure will be raised.
302 if (!mlx5_sys_mem.init) {
304 mlx5_sys_mem.enable = 1;
305 mlx5_sys_mem.init = 1;
306 DRV_LOG(INFO, "%s is selected.", sys_mem_en ? "SYS_MEM" : "RTE_MEM");
307 } else if (mlx5_sys_mem.enable != sys_mem_en) {
308 DRV_LOG(WARNING, "%s is already selected.",
309 mlx5_sys_mem.enable ? "SYS_MEM" : "RTE_MEM");