1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
13 #include <rte_common.h>
14 #include <rte_eal_paging.h>
15 #include <rte_errno.h>
17 #include <rte_memory.h>
18 #include <rte_spinlock.h>
19 #include <rte_tailq.h>
21 #include "eal_filesystem.h"
22 #include "eal_private.h"
24 #include "rte_fbarray.h"
26 #define MASK_SHIFT 6ULL
27 #define MASK_ALIGN (1ULL << MASK_SHIFT)
28 #define MASK_LEN_TO_IDX(x) ((x) >> MASK_SHIFT)
29 #define MASK_LEN_TO_MOD(x) ((x) - RTE_ALIGN_FLOOR(x, MASK_ALIGN))
30 #define MASK_GET_IDX(idx, mod) ((idx << MASK_SHIFT) + mod)
33 * We use this to keep track of created/attached memory areas to prevent user
34 * errors in API usage.
37 TAILQ_ENTRY(mem_area) next;
42 TAILQ_HEAD(mem_area_head, mem_area);
43 /* local per-process tailq */
44 static struct mem_area_head mem_area_tailq =
45 TAILQ_HEAD_INITIALIZER(mem_area_tailq);
46 static rte_spinlock_t mem_area_lock = RTE_SPINLOCK_INITIALIZER;
49 * This is a mask that is always stored at the end of array, to provide fast
50 * way of finding free/used spots without looping through each element.
59 calc_mask_size(unsigned int len)
61 /* mask must be multiple of MASK_ALIGN, even though length of array
62 * itself may not be aligned on that boundary.
64 len = RTE_ALIGN_CEIL(len, MASK_ALIGN);
65 return sizeof(struct used_mask) +
66 sizeof(uint64_t) * MASK_LEN_TO_IDX(len);
70 calc_data_size(size_t page_sz, unsigned int elt_sz, unsigned int len)
72 size_t data_sz = elt_sz * len;
73 size_t msk_sz = calc_mask_size(len);
74 return RTE_ALIGN_CEIL(data_sz + msk_sz, page_sz);
77 static struct used_mask *
78 get_used_mask(void *data, unsigned int elt_sz, unsigned int len)
80 return (struct used_mask *) RTE_PTR_ADD(data, elt_sz * len);
84 resize_and_map(int fd, const char *path, void *addr, size_t len)
88 if (eal_file_truncate(fd, len)) {
89 RTE_LOG(ERR, EAL, "Cannot truncate %s\n", path);
93 map_addr = rte_mem_map(addr, len, RTE_PROT_READ | RTE_PROT_WRITE,
94 RTE_MAP_SHARED | RTE_MAP_FORCE_ADDRESS, fd, 0);
95 if (map_addr != addr) {
102 overlap(const struct mem_area *ma, const void *start, size_t len)
104 const void *end = RTE_PTR_ADD(start, len);
105 const void *ma_start = ma->addr;
106 const void *ma_end = RTE_PTR_ADD(ma->addr, ma->len);
109 if (start >= ma_start && start < ma_end)
112 if (end > ma_start && end < ma_end)
118 find_next_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
121 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
123 unsigned int msk_idx, lookahead_idx, first, first_mod;
124 unsigned int last, last_mod;
125 uint64_t last_msk, ignore_msk;
128 * mask only has granularity of MASK_ALIGN, but start may not be aligned
129 * on that boundary, so construct a special mask to exclude anything we
130 * don't want to see to avoid confusing ctz.
132 first = MASK_LEN_TO_IDX(start);
133 first_mod = MASK_LEN_TO_MOD(start);
134 ignore_msk = ~((1ULL << first_mod) - 1);
136 /* array length may not be aligned, so calculate ignore mask for last
139 last = MASK_LEN_TO_IDX(arr->len);
140 last_mod = MASK_LEN_TO_MOD(arr->len);
141 last_msk = ~(UINT64_MAX << last_mod);
143 for (msk_idx = first; msk_idx < msk->n_masks; msk_idx++) {
144 uint64_t cur_msk, lookahead_msk;
145 unsigned int run_start, clz, left;
148 * The process of getting n consecutive bits for arbitrary n is
149 * a bit involved, but here it is in a nutshell:
151 * 1. let n be the number of consecutive bits we're looking for
152 * 2. check if n can fit in one mask, and if so, do n-1
153 * rshift-ands to see if there is an appropriate run inside
155 * 2a. if we found a run, bail out early
156 * 2b. if we didn't find a run, proceed
157 * 3. invert the mask and count leading zeroes (that is, count
158 * how many consecutive set bits we had starting from the
159 * end of current mask) as k
160 * 3a. if k is 0, continue to next mask
161 * 3b. if k is not 0, we have a potential run
162 * 4. to satisfy our requirements, next mask must have n-k
163 * consecutive set bits right at the start, so we will do
164 * (n-k-1) rshift-ands and check if first bit is set.
166 * Step 4 will need to be repeated if (n-k) > MASK_ALIGN until
167 * we either run out of masks, lose the run, or find what we
170 cur_msk = msk->data[msk_idx];
173 /* if we're looking for free spaces, invert the mask */
177 /* combine current ignore mask with last index ignore mask */
179 ignore_msk |= last_msk;
181 /* if we have an ignore mask, ignore once */
183 cur_msk &= ignore_msk;
187 /* if n can fit in within a single mask, do a search */
188 if (n <= MASK_ALIGN) {
189 uint64_t tmp_msk = cur_msk;
191 for (s_idx = 0; s_idx < n - 1; s_idx++)
192 tmp_msk &= tmp_msk >> 1ULL;
193 /* we found what we were looking for */
195 run_start = __builtin_ctzll(tmp_msk);
196 return MASK_GET_IDX(msk_idx, run_start);
201 * we didn't find our run within the mask, or n > MASK_ALIGN,
202 * so we're going for plan B.
205 /* count leading zeroes on inverted mask */
207 clz = sizeof(cur_msk) * 8;
209 clz = __builtin_clzll(~cur_msk);
211 /* if there aren't any runs at the end either, just continue */
215 /* we have a partial run at the end, so try looking ahead */
216 run_start = MASK_ALIGN - clz;
219 for (lookahead_idx = msk_idx + 1; lookahead_idx < msk->n_masks;
221 unsigned int s_idx, need;
222 lookahead_msk = msk->data[lookahead_idx];
224 /* if we're looking for free space, invert the mask */
226 lookahead_msk = ~lookahead_msk;
228 /* figure out how many consecutive bits we need here */
229 need = RTE_MIN(left, MASK_ALIGN);
231 for (s_idx = 0; s_idx < need - 1; s_idx++)
232 lookahead_msk &= lookahead_msk >> 1ULL;
234 /* if first bit is not set, we've lost the run */
235 if ((lookahead_msk & 1) == 0) {
237 * we've scanned this far, so we know there are
238 * no runs in the space we've lookahead-scanned
239 * as well, so skip that on next iteration.
241 ignore_msk = ~((1ULL << need) - 1);
242 msk_idx = lookahead_idx;
248 /* check if we've found what we were looking for */
255 /* we didn't find anything, so continue */
259 return MASK_GET_IDX(msk_idx, run_start);
261 /* we didn't find anything */
262 rte_errno = used ? ENOENT : ENOSPC;
267 find_next(const struct rte_fbarray *arr, unsigned int start, bool used)
269 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
271 unsigned int idx, first, first_mod;
272 unsigned int last, last_mod;
273 uint64_t last_msk, ignore_msk;
276 * mask only has granularity of MASK_ALIGN, but start may not be aligned
277 * on that boundary, so construct a special mask to exclude anything we
278 * don't want to see to avoid confusing ctz.
280 first = MASK_LEN_TO_IDX(start);
281 first_mod = MASK_LEN_TO_MOD(start);
282 ignore_msk = ~((1ULL << first_mod) - 1ULL);
284 /* array length may not be aligned, so calculate ignore mask for last
287 last = MASK_LEN_TO_IDX(arr->len);
288 last_mod = MASK_LEN_TO_MOD(arr->len);
289 last_msk = ~(-(1ULL) << last_mod);
291 for (idx = first; idx < msk->n_masks; idx++) {
292 uint64_t cur = msk->data[idx];
295 /* if we're looking for free entries, invert mask */
302 /* ignore everything before start on first iteration */
306 /* check if we have any entries */
311 * find first set bit - that will correspond to whatever it is
312 * that we're looking for.
314 found = __builtin_ctzll(cur);
315 return MASK_GET_IDX(idx, found);
317 /* we didn't find anything */
318 rte_errno = used ? ENOENT : ENOSPC;
323 find_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
325 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
327 unsigned int idx, first, first_mod;
328 unsigned int last, last_mod;
330 unsigned int need_len, result = 0;
332 /* array length may not be aligned, so calculate ignore mask for last
335 last = MASK_LEN_TO_IDX(arr->len);
336 last_mod = MASK_LEN_TO_MOD(arr->len);
337 last_msk = ~(-(1ULL) << last_mod);
339 first = MASK_LEN_TO_IDX(start);
340 first_mod = MASK_LEN_TO_MOD(start);
341 for (idx = first; idx < msk->n_masks; idx++, result += need_len) {
342 uint64_t cur = msk->data[idx];
343 unsigned int run_len;
345 need_len = MASK_ALIGN;
347 /* if we're looking for free entries, invert mask */
351 /* if this is last mask, ignore everything after last bit */
355 /* ignore everything before start on first iteration */
358 /* at the start, we don't need the full mask len */
359 need_len -= first_mod;
362 /* we will be looking for zeroes, so invert the mask */
365 /* if mask is zero, we have a complete run */
370 * see if current run ends before mask end.
372 run_len = __builtin_ctzll(cur);
374 /* add however many zeroes we've had in the last run and quit */
375 if (run_len < need_len) {
384 find_prev_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
387 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
389 unsigned int msk_idx, lookbehind_idx, first, first_mod;
393 * mask only has granularity of MASK_ALIGN, but start may not be aligned
394 * on that boundary, so construct a special mask to exclude anything we
395 * don't want to see to avoid confusing ctz.
397 first = MASK_LEN_TO_IDX(start);
398 first_mod = MASK_LEN_TO_MOD(start);
399 /* we're going backwards, so mask must start from the top */
400 ignore_msk = first_mod == MASK_ALIGN - 1 ?
401 UINT64_MAX : /* prevent overflow */
402 ~(UINT64_MAX << (first_mod + 1));
404 /* go backwards, include zero */
407 uint64_t cur_msk, lookbehind_msk;
408 unsigned int run_start, run_end, ctz, left;
411 * The process of getting n consecutive bits from the top for
412 * arbitrary n is a bit involved, but here it is in a nutshell:
414 * 1. let n be the number of consecutive bits we're looking for
415 * 2. check if n can fit in one mask, and if so, do n-1
416 * lshift-ands to see if there is an appropriate run inside
418 * 2a. if we found a run, bail out early
419 * 2b. if we didn't find a run, proceed
420 * 3. invert the mask and count trailing zeroes (that is, count
421 * how many consecutive set bits we had starting from the
422 * start of current mask) as k
423 * 3a. if k is 0, continue to next mask
424 * 3b. if k is not 0, we have a potential run
425 * 4. to satisfy our requirements, next mask must have n-k
426 * consecutive set bits at the end, so we will do (n-k-1)
427 * lshift-ands and check if last bit is set.
429 * Step 4 will need to be repeated if (n-k) > MASK_ALIGN until
430 * we either run out of masks, lose the run, or find what we
433 cur_msk = msk->data[msk_idx];
436 /* if we're looking for free spaces, invert the mask */
440 /* if we have an ignore mask, ignore once */
442 cur_msk &= ignore_msk;
446 /* if n can fit in within a single mask, do a search */
447 if (n <= MASK_ALIGN) {
448 uint64_t tmp_msk = cur_msk;
450 for (s_idx = 0; s_idx < n - 1; s_idx++)
451 tmp_msk &= tmp_msk << 1ULL;
452 /* we found what we were looking for */
454 /* clz will give us offset from end of mask, and
455 * we only get the end of our run, not start,
456 * so adjust result to point to where start
459 run_start = MASK_ALIGN -
460 __builtin_clzll(tmp_msk) - n;
461 return MASK_GET_IDX(msk_idx, run_start);
466 * we didn't find our run within the mask, or n > MASK_ALIGN,
467 * so we're going for plan B.
470 /* count trailing zeroes on inverted mask */
472 ctz = sizeof(cur_msk) * 8;
474 ctz = __builtin_ctzll(~cur_msk);
476 /* if there aren't any runs at the start either, just
482 /* we have a partial run at the start, so try looking behind */
483 run_end = MASK_GET_IDX(msk_idx, ctz);
486 /* go backwards, include zero */
487 lookbehind_idx = msk_idx - 1;
489 /* we can't lookbehind as we've run out of masks, so stop */
494 const uint64_t last_bit = 1ULL << (MASK_ALIGN - 1);
495 unsigned int s_idx, need;
497 lookbehind_msk = msk->data[lookbehind_idx];
499 /* if we're looking for free space, invert the mask */
501 lookbehind_msk = ~lookbehind_msk;
503 /* figure out how many consecutive bits we need here */
504 need = RTE_MIN(left, MASK_ALIGN);
506 for (s_idx = 0; s_idx < need - 1; s_idx++)
507 lookbehind_msk &= lookbehind_msk << 1ULL;
509 /* if last bit is not set, we've lost the run */
510 if ((lookbehind_msk & last_bit) == 0) {
512 * we've scanned this far, so we know there are
513 * no runs in the space we've lookbehind-scanned
514 * as well, so skip that on next iteration.
516 ignore_msk = UINT64_MAX << need;
517 msk_idx = lookbehind_idx;
523 /* check if we've found what we were looking for */
528 } while ((lookbehind_idx--) != 0); /* decrement after check to
532 /* we didn't find anything, so continue */
536 /* we've found what we were looking for, but we only know where
537 * the run ended, so calculate start position.
540 } while (msk_idx-- != 0); /* decrement after check to include zero */
541 /* we didn't find anything */
542 rte_errno = used ? ENOENT : ENOSPC;
547 find_prev(const struct rte_fbarray *arr, unsigned int start, bool used)
549 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
551 unsigned int idx, first, first_mod;
555 * mask only has granularity of MASK_ALIGN, but start may not be aligned
556 * on that boundary, so construct a special mask to exclude anything we
557 * don't want to see to avoid confusing clz.
559 first = MASK_LEN_TO_IDX(start);
560 first_mod = MASK_LEN_TO_MOD(start);
561 /* we're going backwards, so mask must start from the top */
562 ignore_msk = first_mod == MASK_ALIGN - 1 ?
563 UINT64_MAX : /* prevent overflow */
564 ~(UINT64_MAX << (first_mod + 1));
566 /* go backwards, include zero */
569 uint64_t cur = msk->data[idx];
572 /* if we're looking for free entries, invert mask */
576 /* ignore everything before start on first iteration */
580 /* check if we have any entries */
585 * find last set bit - that will correspond to whatever it is
586 * that we're looking for. we're counting trailing zeroes, thus
587 * the value we get is counted from end of mask, so calculate
588 * position from start of mask.
590 found = MASK_ALIGN - __builtin_clzll(cur) - 1;
592 return MASK_GET_IDX(idx, found);
593 } while (idx-- != 0); /* decrement after check to include zero*/
595 /* we didn't find anything */
596 rte_errno = used ? ENOENT : ENOSPC;
601 find_rev_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
603 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
605 unsigned int idx, first, first_mod;
606 unsigned int need_len, result = 0;
608 first = MASK_LEN_TO_IDX(start);
609 first_mod = MASK_LEN_TO_MOD(start);
611 /* go backwards, include zero */
614 uint64_t cur = msk->data[idx];
615 unsigned int run_len;
617 need_len = MASK_ALIGN;
619 /* if we're looking for free entries, invert mask */
623 /* ignore everything after start on first iteration */
625 unsigned int end_len = MASK_ALIGN - first_mod - 1;
627 /* at the start, we don't need the full mask len */
631 /* we will be looking for zeroes, so invert the mask */
634 /* if mask is zero, we have a complete run */
639 * see where run ends, starting from the end.
641 run_len = __builtin_clzll(cur);
643 /* add however many zeroes we've had in the last run and quit */
644 if (run_len < need_len) {
650 } while (idx-- != 0); /* decrement after check to include zero */
655 set_used(struct rte_fbarray *arr, unsigned int idx, bool used)
657 struct used_mask *msk;
658 uint64_t msk_bit = 1ULL << MASK_LEN_TO_MOD(idx);
659 unsigned int msk_idx = MASK_LEN_TO_IDX(idx);
663 if (arr == NULL || idx >= arr->len) {
667 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
670 /* prevent array from changing under us */
671 rte_rwlock_write_lock(&arr->rwlock);
673 already_used = (msk->data[msk_idx] & msk_bit) != 0;
675 /* nothing to be done */
676 if (used == already_used)
680 msk->data[msk_idx] |= msk_bit;
683 msk->data[msk_idx] &= ~msk_bit;
687 rte_rwlock_write_unlock(&arr->rwlock);
693 fully_validate(const char *name, unsigned int elt_sz, unsigned int len)
695 if (name == NULL || elt_sz == 0 || len == 0 || len > INT_MAX) {
700 if (strnlen(name, RTE_FBARRAY_NAME_LEN) == RTE_FBARRAY_NAME_LEN) {
701 rte_errno = ENAMETOOLONG;
708 rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,
711 size_t page_sz, mmap_len;
713 struct used_mask *msk;
714 struct mem_area *ma = NULL;
717 const struct internal_config *internal_conf =
718 eal_get_internal_configuration();
725 if (fully_validate(name, elt_sz, len))
728 /* allocate mem area before doing anything */
729 ma = malloc(sizeof(*ma));
735 page_sz = rte_mem_page_size();
736 if (page_sz == (size_t)-1) {
741 /* calculate our memory limits */
742 mmap_len = calc_data_size(page_sz, elt_sz, len);
744 data = eal_get_virtual_area(NULL, &mmap_len, page_sz, 0, 0);
750 rte_spinlock_lock(&mem_area_lock);
754 if (internal_conf->no_shconf) {
755 /* remap virtual area as writable */
756 static const int flags = RTE_MAP_FORCE_ADDRESS |
757 RTE_MAP_PRIVATE | RTE_MAP_ANONYMOUS;
758 void *new_data = rte_mem_map(data, mmap_len,
759 RTE_PROT_READ | RTE_PROT_WRITE, flags, fd, 0);
760 if (new_data == NULL) {
761 RTE_LOG(DEBUG, EAL, "%s(): couldn't remap anonymous memory: %s\n",
762 __func__, rte_strerror(rte_errno));
766 eal_get_fbarray_path(path, sizeof(path), name);
769 * Each fbarray is unique to process namespace, i.e. the
770 * filename depends on process prefix. Try to take out a lock
771 * and see if we succeed. If we don't, someone else is using it
774 fd = eal_file_open(path, EAL_OPEN_CREATE | EAL_OPEN_READWRITE);
776 RTE_LOG(DEBUG, EAL, "%s(): couldn't open %s: %s\n",
777 __func__, path, rte_strerror(rte_errno));
779 } else if (eal_file_lock(
780 fd, EAL_FLOCK_EXCLUSIVE, EAL_FLOCK_RETURN)) {
781 RTE_LOG(DEBUG, EAL, "%s(): couldn't lock %s: %s\n",
782 __func__, path, rte_strerror(rte_errno));
787 /* take out a non-exclusive lock, so that other processes could
788 * still attach to it, but no other process could reinitialize
791 if (eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN))
794 if (resize_and_map(fd, path, data, mmap_len))
801 /* do not close fd - keep it until detach/destroy */
802 TAILQ_INSERT_TAIL(&mem_area_tailq, ma, next);
804 /* initialize the data */
805 memset(data, 0, mmap_len);
807 /* populate data structure */
808 strlcpy(arr->name, name, sizeof(arr->name));
811 arr->elt_sz = elt_sz;
814 msk = get_used_mask(data, elt_sz, len);
815 msk->n_masks = MASK_LEN_TO_IDX(RTE_ALIGN_CEIL(len, MASK_ALIGN));
817 rte_rwlock_init(&arr->rwlock);
819 rte_spinlock_unlock(&mem_area_lock);
824 rte_mem_unmap(data, mmap_len);
829 rte_spinlock_unlock(&mem_area_lock);
834 rte_fbarray_attach(struct rte_fbarray *arr)
836 struct mem_area *ma = NULL, *tmp = NULL;
837 size_t page_sz, mmap_len;
848 * we don't need to synchronize attach as two values we need (element
849 * size and array length) are constant for the duration of life of
850 * the array, so the parts we care about will not race.
853 if (fully_validate(arr->name, arr->elt_sz, arr->len))
856 ma = malloc(sizeof(*ma));
862 page_sz = rte_mem_page_size();
863 if (page_sz == (size_t)-1) {
868 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
870 /* check the tailq - maybe user has already mapped this address space */
871 rte_spinlock_lock(&mem_area_lock);
873 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
874 if (overlap(tmp, arr->data, mmap_len)) {
880 /* we know this memory area is unique, so proceed */
882 data = eal_get_virtual_area(arr->data, &mmap_len, page_sz, 0, 0);
886 eal_get_fbarray_path(path, sizeof(path), arr->name);
888 fd = eal_file_open(path, EAL_OPEN_READWRITE);
893 /* lock the file, to let others know we're using it */
894 if (eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN))
897 if (resize_and_map(fd, path, data, mmap_len))
900 /* store our new memory area */
902 ma->fd = fd; /* keep fd until detach/destroy */
905 TAILQ_INSERT_TAIL(&mem_area_tailq, ma, next);
909 rte_spinlock_unlock(&mem_area_lock);
913 rte_mem_unmap(data, mmap_len);
917 rte_spinlock_unlock(&mem_area_lock);
922 rte_fbarray_detach(struct rte_fbarray *arr)
924 struct mem_area *tmp = NULL;
934 * we don't need to synchronize detach as two values we need (element
935 * size and total capacity) are constant for the duration of life of
936 * the array, so the parts we care about will not race. if the user is
937 * detaching while doing something else in the same process, we can't
938 * really do anything about it, things will blow up either way.
941 size_t page_sz = rte_mem_page_size();
942 if (page_sz == (size_t)-1)
945 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
947 /* does this area exist? */
948 rte_spinlock_lock(&mem_area_lock);
950 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
951 if (tmp->addr == arr->data && tmp->len == mmap_len)
960 rte_mem_unmap(arr->data, mmap_len);
962 /* area is unmapped, close fd and remove the tailq entry */
965 TAILQ_REMOVE(&mem_area_tailq, tmp, next);
970 rte_spinlock_unlock(&mem_area_lock);
975 rte_fbarray_destroy(struct rte_fbarray *arr)
977 struct mem_area *tmp = NULL;
981 const struct internal_config *internal_conf =
982 eal_get_internal_configuration();
990 * we don't need to synchronize detach as two values we need (element
991 * size and total capacity) are constant for the duration of life of
992 * the array, so the parts we care about will not race. if the user is
993 * detaching while doing something else in the same process, we can't
994 * really do anything about it, things will blow up either way.
997 size_t page_sz = rte_mem_page_size();
998 if (page_sz == (size_t)-1)
1001 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
1003 /* does this area exist? */
1004 rte_spinlock_lock(&mem_area_lock);
1006 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
1007 if (tmp->addr == arr->data && tmp->len == mmap_len)
1015 /* with no shconf, there were never any files to begin with */
1016 if (!internal_conf->no_shconf) {
1018 * attempt to get an exclusive lock on the file, to ensure it
1019 * has been detached by all other processes
1022 if (eal_file_lock(fd, EAL_FLOCK_EXCLUSIVE, EAL_FLOCK_RETURN)) {
1023 RTE_LOG(DEBUG, EAL, "Cannot destroy fbarray - another process is using it\n");
1029 /* we're OK to destroy the file */
1030 eal_get_fbarray_path(path, sizeof(path), arr->name);
1032 RTE_LOG(DEBUG, EAL, "Cannot unlink fbarray: %s\n",
1036 * we're still holding an exclusive lock, so drop it to
1039 eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN);
1046 rte_mem_unmap(arr->data, mmap_len);
1048 /* area is unmapped, remove the tailq entry */
1049 TAILQ_REMOVE(&mem_area_tailq, tmp, next);
1053 /* reset the fbarray structure */
1054 memset(arr, 0, sizeof(*arr));
1056 rte_spinlock_unlock(&mem_area_lock);
1061 rte_fbarray_get(const struct rte_fbarray *arr, unsigned int idx)
1069 if (idx >= arr->len) {
1074 ret = RTE_PTR_ADD(arr->data, idx * arr->elt_sz);
1080 rte_fbarray_set_used(struct rte_fbarray *arr, unsigned int idx)
1082 return set_used(arr, idx, true);
1086 rte_fbarray_set_free(struct rte_fbarray *arr, unsigned int idx)
1088 return set_used(arr, idx, false);
1092 rte_fbarray_is_used(struct rte_fbarray *arr, unsigned int idx)
1094 struct used_mask *msk;
1099 if (arr == NULL || idx >= arr->len) {
1104 /* prevent array from changing under us */
1105 rte_rwlock_read_lock(&arr->rwlock);
1107 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
1108 msk_idx = MASK_LEN_TO_IDX(idx);
1109 msk_bit = 1ULL << MASK_LEN_TO_MOD(idx);
1111 ret = (msk->data[msk_idx] & msk_bit) != 0;
1113 rte_rwlock_read_unlock(&arr->rwlock);
1119 fbarray_find(struct rte_fbarray *arr, unsigned int start, bool next, bool used)
1123 if (arr == NULL || start >= arr->len) {
1128 /* prevent array from changing under us */
1129 rte_rwlock_read_lock(&arr->rwlock);
1131 /* cheap checks to prevent doing useless work */
1133 if (arr->len == arr->count) {
1137 if (arr->count == 0) {
1142 if (arr->count == 0) {
1146 if (arr->len == arr->count) {
1152 ret = find_next(arr, start, used);
1154 ret = find_prev(arr, start, used);
1156 rte_rwlock_read_unlock(&arr->rwlock);
1161 rte_fbarray_find_next_free(struct rte_fbarray *arr, unsigned int start)
1163 return fbarray_find(arr, start, true, false);
1167 rte_fbarray_find_next_used(struct rte_fbarray *arr, unsigned int start)
1169 return fbarray_find(arr, start, true, true);
1173 rte_fbarray_find_prev_free(struct rte_fbarray *arr, unsigned int start)
1175 return fbarray_find(arr, start, false, false);
1179 rte_fbarray_find_prev_used(struct rte_fbarray *arr, unsigned int start)
1181 return fbarray_find(arr, start, false, true);
1185 fbarray_find_n(struct rte_fbarray *arr, unsigned int start, unsigned int n,
1186 bool next, bool used)
1190 if (arr == NULL || start >= arr->len || n > arr->len || n == 0) {
1194 if (next && (arr->len - start) < n) {
1195 rte_errno = used ? ENOENT : ENOSPC;
1198 if (!next && start < (n - 1)) {
1199 rte_errno = used ? ENOENT : ENOSPC;
1203 /* prevent array from changing under us */
1204 rte_rwlock_read_lock(&arr->rwlock);
1206 /* cheap checks to prevent doing useless work */
1208 if (arr->len == arr->count || arr->len - arr->count < n) {
1212 if (arr->count == 0) {
1213 ret = next ? start : start - n + 1;
1217 if (arr->count < n) {
1221 if (arr->count == arr->len) {
1222 ret = next ? start : start - n + 1;
1228 ret = find_next_n(arr, start, n, used);
1230 ret = find_prev_n(arr, start, n, used);
1232 rte_rwlock_read_unlock(&arr->rwlock);
1237 rte_fbarray_find_next_n_free(struct rte_fbarray *arr, unsigned int start,
1240 return fbarray_find_n(arr, start, n, true, false);
1244 rte_fbarray_find_next_n_used(struct rte_fbarray *arr, unsigned int start,
1247 return fbarray_find_n(arr, start, n, true, true);
1251 rte_fbarray_find_prev_n_free(struct rte_fbarray *arr, unsigned int start,
1254 return fbarray_find_n(arr, start, n, false, false);
1258 rte_fbarray_find_prev_n_used(struct rte_fbarray *arr, unsigned int start,
1261 return fbarray_find_n(arr, start, n, false, true);
1265 fbarray_find_contig(struct rte_fbarray *arr, unsigned int start, bool next,
1270 if (arr == NULL || start >= arr->len) {
1275 /* prevent array from changing under us */
1276 rte_rwlock_read_lock(&arr->rwlock);
1278 /* cheap checks to prevent doing useless work */
1280 if (arr->count == 0) {
1284 if (next && arr->count == arr->len) {
1285 ret = arr->len - start;
1288 if (!next && arr->count == arr->len) {
1293 if (arr->len == arr->count) {
1297 if (next && arr->count == 0) {
1298 ret = arr->len - start;
1301 if (!next && arr->count == 0) {
1308 ret = find_contig(arr, start, used);
1310 ret = find_rev_contig(arr, start, used);
1312 rte_rwlock_read_unlock(&arr->rwlock);
1317 fbarray_find_biggest(struct rte_fbarray *arr, unsigned int start, bool used,
1320 int cur_idx, next_idx, cur_len, biggest_idx, biggest_len;
1321 /* don't stack if conditions, use function pointers instead */
1322 int (*find_func)(struct rte_fbarray *, unsigned int);
1323 int (*find_contig_func)(struct rte_fbarray *, unsigned int);
1325 if (arr == NULL || start >= arr->len) {
1329 /* the other API calls already do their fair share of cheap checks, so
1330 * no need to do them here.
1333 /* the API's called are thread-safe, but something may still happen
1334 * between the API calls, so lock the fbarray. all other API's are
1335 * read-locking the fbarray, so read lock here is OK.
1337 rte_rwlock_read_lock(&arr->rwlock);
1339 /* pick out appropriate functions */
1342 find_func = rte_fbarray_find_prev_used;
1343 find_contig_func = rte_fbarray_find_rev_contig_used;
1345 find_func = rte_fbarray_find_next_used;
1346 find_contig_func = rte_fbarray_find_contig_used;
1350 find_func = rte_fbarray_find_prev_free;
1351 find_contig_func = rte_fbarray_find_rev_contig_free;
1353 find_func = rte_fbarray_find_next_free;
1354 find_contig_func = rte_fbarray_find_contig_free;
1359 biggest_idx = -1; /* default is error */
1362 cur_idx = find_func(arr, cur_idx);
1364 /* block found, check its length */
1366 cur_len = find_contig_func(arr, cur_idx);
1367 /* decide where we go next */
1368 next_idx = rev ? cur_idx - cur_len : cur_idx + cur_len;
1369 /* move current index to start of chunk */
1370 cur_idx = rev ? next_idx + 1 : cur_idx;
1372 if (cur_len > biggest_len) {
1373 biggest_idx = cur_idx;
1374 biggest_len = cur_len;
1377 /* in reverse mode, next_idx may be -1 if chunk started
1378 * at array beginning. this means there's no more work
1384 /* nothing more to find, stop. however, a failed API
1385 * call has set rte_errno, which we want to ignore, as
1386 * reaching the end of fbarray is not an error.
1392 /* if we didn't find anything at all, set rte_errno */
1393 if (biggest_idx < 0)
1394 rte_errno = used ? ENOENT : ENOSPC;
1396 rte_rwlock_read_unlock(&arr->rwlock);
1401 rte_fbarray_find_biggest_free(struct rte_fbarray *arr, unsigned int start)
1403 return fbarray_find_biggest(arr, start, false, false);
1407 rte_fbarray_find_biggest_used(struct rte_fbarray *arr, unsigned int start)
1409 return fbarray_find_biggest(arr, start, true, false);
1413 rte_fbarray_find_rev_biggest_free(struct rte_fbarray *arr, unsigned int start)
1415 return fbarray_find_biggest(arr, start, false, true);
1419 rte_fbarray_find_rev_biggest_used(struct rte_fbarray *arr, unsigned int start)
1421 return fbarray_find_biggest(arr, start, true, true);
1426 rte_fbarray_find_contig_free(struct rte_fbarray *arr, unsigned int start)
1428 return fbarray_find_contig(arr, start, true, false);
1432 rte_fbarray_find_contig_used(struct rte_fbarray *arr, unsigned int start)
1434 return fbarray_find_contig(arr, start, true, true);
1438 rte_fbarray_find_rev_contig_free(struct rte_fbarray *arr, unsigned int start)
1440 return fbarray_find_contig(arr, start, false, false);
1444 rte_fbarray_find_rev_contig_used(struct rte_fbarray *arr, unsigned int start)
1446 return fbarray_find_contig(arr, start, false, true);
1450 rte_fbarray_find_idx(const struct rte_fbarray *arr, const void *elt)
1456 * no need to synchronize as it doesn't matter if underlying data
1457 * changes - we're doing pointer arithmetic here.
1460 if (arr == NULL || elt == NULL) {
1464 end = RTE_PTR_ADD(arr->data, arr->elt_sz * arr->len);
1465 if (elt < arr->data || elt >= end) {
1470 ret = RTE_PTR_DIFF(elt, arr->data) / arr->elt_sz;
1476 rte_fbarray_dump_metadata(struct rte_fbarray *arr, FILE *f)
1478 struct used_mask *msk;
1481 if (arr == NULL || f == NULL) {
1486 if (fully_validate(arr->name, arr->elt_sz, arr->len)) {
1487 fprintf(f, "Invalid file-backed array\n");
1491 /* prevent array from changing under us */
1492 rte_rwlock_read_lock(&arr->rwlock);
1494 fprintf(f, "File-backed array: %s\n", arr->name);
1495 fprintf(f, "size: %i occupied: %i elt_sz: %i\n",
1496 arr->len, arr->count, arr->elt_sz);
1498 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
1500 for (i = 0; i < msk->n_masks; i++)
1501 fprintf(f, "msk idx %i: 0x%016" PRIx64 "\n", i, msk->data[i]);
1503 rte_rwlock_read_unlock(&arr->rwlock);