1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
9 #include <rte_fbarray.h>
10 #include <rte_memzone.h>
11 #include <rte_memory.h>
12 #include <rte_eal_memconfig.h>
13 #include <rte_rwlock.h>
15 #include "eal_private.h"
16 #include "eal_internal_cfg.h"
17 #include "eal_memalloc.h"
19 struct mem_event_callback_entry {
20 TAILQ_ENTRY(mem_event_callback_entry) next;
21 char name[RTE_MEM_EVENT_CALLBACK_NAME_LEN];
22 rte_mem_event_callback_t clb;
25 struct mem_alloc_validator_entry {
26 TAILQ_ENTRY(mem_alloc_validator_entry) next;
27 char name[RTE_MEM_ALLOC_VALIDATOR_NAME_LEN];
28 rte_mem_alloc_validator_t clb;
33 /** Double linked list of actions. */
34 TAILQ_HEAD(mem_event_callback_entry_list, mem_event_callback_entry);
35 TAILQ_HEAD(mem_alloc_validator_entry_list, mem_alloc_validator_entry);
37 static struct mem_event_callback_entry_list mem_event_callback_list =
38 TAILQ_HEAD_INITIALIZER(mem_event_callback_list);
39 static rte_rwlock_t mem_event_rwlock = RTE_RWLOCK_INITIALIZER;
41 static struct mem_alloc_validator_entry_list mem_alloc_validator_list =
42 TAILQ_HEAD_INITIALIZER(mem_alloc_validator_list);
43 static rte_rwlock_t mem_alloc_validator_rwlock = RTE_RWLOCK_INITIALIZER;
45 static struct mem_event_callback_entry *
46 find_mem_event_callback(const char *name)
48 struct mem_event_callback_entry *r;
50 TAILQ_FOREACH(r, &mem_event_callback_list, next) {
51 if (!strcmp(r->name, name))
57 static struct mem_alloc_validator_entry *
58 find_mem_alloc_validator(const char *name, int socket_id)
60 struct mem_alloc_validator_entry *r;
62 TAILQ_FOREACH(r, &mem_alloc_validator_list, next) {
63 if (!strcmp(r->name, name) && r->socket_id == socket_id)
70 eal_memalloc_is_contig(const struct rte_memseg_list *msl, void *start,
73 void *end, *aligned_start, *aligned_end;
74 size_t pgsz = (size_t)msl->page_sz;
75 const struct rte_memseg *ms;
77 /* for IOVA_VA, it's always contiguous */
78 if (rte_eal_iova_mode() == RTE_IOVA_VA)
81 /* for legacy memory, it's always contiguous */
82 if (internal_config.legacy_mem)
85 end = RTE_PTR_ADD(start, len);
87 /* for nohuge, we check pagemap, otherwise check memseg */
88 if (!rte_eal_has_hugepages()) {
89 rte_iova_t cur, expected;
91 aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz);
92 aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz);
94 /* if start and end are on the same page, bail out early */
95 if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz)
98 /* skip first iteration */
99 cur = rte_mem_virt2iova(aligned_start);
100 expected = cur + pgsz;
101 aligned_start = RTE_PTR_ADD(aligned_start, pgsz);
103 while (aligned_start < aligned_end) {
104 cur = rte_mem_virt2iova(aligned_start);
107 aligned_start = RTE_PTR_ADD(aligned_start, pgsz);
111 int start_seg, end_seg, cur_seg;
112 rte_iova_t cur, expected;
114 aligned_start = RTE_PTR_ALIGN_FLOOR(start, pgsz);
115 aligned_end = RTE_PTR_ALIGN_CEIL(end, pgsz);
117 start_seg = RTE_PTR_DIFF(aligned_start, msl->base_va) /
119 end_seg = RTE_PTR_DIFF(aligned_end, msl->base_va) /
122 /* if start and end are on the same page, bail out early */
123 if (RTE_PTR_DIFF(aligned_end, aligned_start) == pgsz)
126 /* skip first iteration */
127 ms = rte_fbarray_get(&msl->memseg_arr, start_seg);
129 expected = cur + pgsz;
131 /* if we can't access IOVA addresses, assume non-contiguous */
132 if (cur == RTE_BAD_IOVA)
135 for (cur_seg = start_seg + 1; cur_seg < end_seg;
136 cur_seg++, expected += pgsz) {
137 ms = rte_fbarray_get(&msl->memseg_arr, cur_seg);
139 if (ms->iova != expected)
147 eal_memalloc_mem_event_callback_register(const char *name,
148 rte_mem_event_callback_t clb)
150 struct mem_event_callback_entry *entry;
152 if (name == NULL || clb == NULL) {
156 len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
160 } else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
161 rte_errno = ENAMETOOLONG;
164 rte_rwlock_write_lock(&mem_event_rwlock);
166 entry = find_mem_event_callback(name);
173 entry = malloc(sizeof(*entry));
180 /* callback successfully created and is valid, add it to the list */
182 snprintf(entry->name, RTE_MEM_EVENT_CALLBACK_NAME_LEN, "%s", name);
183 TAILQ_INSERT_TAIL(&mem_event_callback_list, entry, next);
187 RTE_LOG(DEBUG, EAL, "Mem event callback '%s' registered\n", name);
190 rte_rwlock_write_unlock(&mem_event_rwlock);
195 eal_memalloc_mem_event_callback_unregister(const char *name)
197 struct mem_event_callback_entry *entry;
204 len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
208 } else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
209 rte_errno = ENAMETOOLONG;
212 rte_rwlock_write_lock(&mem_event_rwlock);
214 entry = find_mem_event_callback(name);
220 TAILQ_REMOVE(&mem_event_callback_list, entry, next);
225 RTE_LOG(DEBUG, EAL, "Mem event callback '%s' unregistered\n", name);
228 rte_rwlock_write_unlock(&mem_event_rwlock);
233 eal_memalloc_mem_event_notify(enum rte_mem_event event, const void *start,
236 struct mem_event_callback_entry *entry;
238 rte_rwlock_read_lock(&mem_event_rwlock);
240 TAILQ_FOREACH(entry, &mem_event_callback_list, next) {
241 RTE_LOG(DEBUG, EAL, "Calling mem event callback %s",
243 entry->clb(event, start, len);
246 rte_rwlock_read_unlock(&mem_event_rwlock);
250 eal_memalloc_mem_alloc_validator_register(const char *name,
251 rte_mem_alloc_validator_t clb, int socket_id, size_t limit)
253 struct mem_alloc_validator_entry *entry;
255 if (name == NULL || clb == NULL || socket_id < 0) {
259 len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
263 } else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) {
264 rte_errno = ENAMETOOLONG;
267 rte_rwlock_write_lock(&mem_alloc_validator_rwlock);
269 entry = find_mem_alloc_validator(name, socket_id);
276 entry = malloc(sizeof(*entry));
283 /* callback successfully created and is valid, add it to the list */
285 entry->socket_id = socket_id;
286 entry->limit = limit;
287 snprintf(entry->name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN, "%s", name);
288 TAILQ_INSERT_TAIL(&mem_alloc_validator_list, entry, next);
292 RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i with limit %zu registered\n",
293 name, socket_id, limit);
296 rte_rwlock_write_unlock(&mem_alloc_validator_rwlock);
301 eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id)
303 struct mem_alloc_validator_entry *entry;
306 if (name == NULL || socket_id < 0) {
310 len = strnlen(name, RTE_MEM_ALLOC_VALIDATOR_NAME_LEN);
314 } else if (len == RTE_MEM_ALLOC_VALIDATOR_NAME_LEN) {
315 rte_errno = ENAMETOOLONG;
318 rte_rwlock_write_lock(&mem_alloc_validator_rwlock);
320 entry = find_mem_alloc_validator(name, socket_id);
326 TAILQ_REMOVE(&mem_alloc_validator_list, entry, next);
331 RTE_LOG(DEBUG, EAL, "Mem alloc validator '%s' on socket %i unregistered\n",
335 rte_rwlock_write_unlock(&mem_alloc_validator_rwlock);
340 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len)
342 struct mem_alloc_validator_entry *entry;
345 rte_rwlock_read_lock(&mem_alloc_validator_rwlock);
347 TAILQ_FOREACH(entry, &mem_alloc_validator_list, next) {
348 if (entry->socket_id != socket_id || entry->limit > new_len)
350 RTE_LOG(DEBUG, EAL, "Calling mem alloc validator '%s' on socket %i\n",
351 entry->name, entry->socket_id);
352 if (entry->clb(socket_id, entry->limit, new_len) < 0)
356 rte_rwlock_read_unlock(&mem_alloc_validator_rwlock);