1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
13 #include <rte_common.h>
14 #include <rte_eal_paging.h>
15 #include <rte_errno.h>
17 #include <rte_memory.h>
18 #include <rte_spinlock.h>
19 #include <rte_tailq.h>
21 #include "eal_filesystem.h"
22 #include "eal_private.h"
24 #include "rte_fbarray.h"
26 #define MASK_SHIFT 6ULL
27 #define MASK_ALIGN (1ULL << MASK_SHIFT)
28 #define MASK_LEN_TO_IDX(x) ((x) >> MASK_SHIFT)
29 #define MASK_LEN_TO_MOD(x) ((x) - RTE_ALIGN_FLOOR(x, MASK_ALIGN))
30 #define MASK_GET_IDX(idx, mod) ((idx << MASK_SHIFT) + mod)
33 * We use this to keep track of created/attached memory areas to prevent user
34 * errors in API usage.
37 TAILQ_ENTRY(mem_area) next;
42 TAILQ_HEAD(mem_area_head, mem_area);
43 /* local per-process tailq */
44 static struct mem_area_head mem_area_tailq =
45 TAILQ_HEAD_INITIALIZER(mem_area_tailq);
46 static rte_spinlock_t mem_area_lock = RTE_SPINLOCK_INITIALIZER;
49 * This is a mask that is always stored at the end of array, to provide fast
50 * way of finding free/used spots without looping through each element.
59 calc_mask_size(unsigned int len)
61 /* mask must be multiple of MASK_ALIGN, even though length of array
62 * itself may not be aligned on that boundary.
64 len = RTE_ALIGN_CEIL(len, MASK_ALIGN);
65 return sizeof(struct used_mask) +
66 sizeof(uint64_t) * MASK_LEN_TO_IDX(len);
70 calc_data_size(size_t page_sz, unsigned int elt_sz, unsigned int len)
72 size_t data_sz = elt_sz * len;
73 size_t msk_sz = calc_mask_size(len);
74 return RTE_ALIGN_CEIL(data_sz + msk_sz, page_sz);
77 static struct used_mask *
78 get_used_mask(void *data, unsigned int elt_sz, unsigned int len)
80 return (struct used_mask *) RTE_PTR_ADD(data, elt_sz * len);
84 resize_and_map(int fd, void *addr, size_t len)
89 if (eal_file_truncate(fd, len)) {
90 RTE_LOG(ERR, EAL, "Cannot truncate %s\n", path);
94 map_addr = rte_mem_map(addr, len, RTE_PROT_READ | RTE_PROT_WRITE,
95 RTE_MAP_SHARED | RTE_MAP_FORCE_ADDRESS, fd, 0);
96 if (map_addr != addr) {
103 overlap(const struct mem_area *ma, const void *start, size_t len)
105 const void *end = RTE_PTR_ADD(start, len);
106 const void *ma_start = ma->addr;
107 const void *ma_end = RTE_PTR_ADD(ma->addr, ma->len);
110 if (start >= ma_start && start < ma_end)
113 if (end >= ma_start && end < ma_end)
119 find_next_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
122 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
124 unsigned int msk_idx, lookahead_idx, first, first_mod;
125 unsigned int last, last_mod;
126 uint64_t last_msk, ignore_msk;
129 * mask only has granularity of MASK_ALIGN, but start may not be aligned
130 * on that boundary, so construct a special mask to exclude anything we
131 * don't want to see to avoid confusing ctz.
133 first = MASK_LEN_TO_IDX(start);
134 first_mod = MASK_LEN_TO_MOD(start);
135 ignore_msk = ~((1ULL << first_mod) - 1);
137 /* array length may not be aligned, so calculate ignore mask for last
140 last = MASK_LEN_TO_IDX(arr->len);
141 last_mod = MASK_LEN_TO_MOD(arr->len);
142 last_msk = ~(-1ULL << last_mod);
144 for (msk_idx = first; msk_idx < msk->n_masks; msk_idx++) {
145 uint64_t cur_msk, lookahead_msk;
146 unsigned int run_start, clz, left;
149 * The process of getting n consecutive bits for arbitrary n is
150 * a bit involved, but here it is in a nutshell:
152 * 1. let n be the number of consecutive bits we're looking for
153 * 2. check if n can fit in one mask, and if so, do n-1
154 * rshift-ands to see if there is an appropriate run inside
156 * 2a. if we found a run, bail out early
157 * 2b. if we didn't find a run, proceed
158 * 3. invert the mask and count leading zeroes (that is, count
159 * how many consecutive set bits we had starting from the
160 * end of current mask) as k
161 * 3a. if k is 0, continue to next mask
162 * 3b. if k is not 0, we have a potential run
163 * 4. to satisfy our requirements, next mask must have n-k
164 * consecutive set bits right at the start, so we will do
165 * (n-k-1) rshift-ands and check if first bit is set.
167 * Step 4 will need to be repeated if (n-k) > MASK_ALIGN until
168 * we either run out of masks, lose the run, or find what we
171 cur_msk = msk->data[msk_idx];
174 /* if we're looking for free spaces, invert the mask */
178 /* combine current ignore mask with last index ignore mask */
180 ignore_msk |= last_msk;
182 /* if we have an ignore mask, ignore once */
184 cur_msk &= ignore_msk;
188 /* if n can fit in within a single mask, do a search */
189 if (n <= MASK_ALIGN) {
190 uint64_t tmp_msk = cur_msk;
192 for (s_idx = 0; s_idx < n - 1; s_idx++)
193 tmp_msk &= tmp_msk >> 1ULL;
194 /* we found what we were looking for */
196 run_start = __builtin_ctzll(tmp_msk);
197 return MASK_GET_IDX(msk_idx, run_start);
202 * we didn't find our run within the mask, or n > MASK_ALIGN,
203 * so we're going for plan B.
206 /* count leading zeroes on inverted mask */
208 clz = sizeof(cur_msk) * 8;
210 clz = __builtin_clzll(~cur_msk);
212 /* if there aren't any runs at the end either, just continue */
216 /* we have a partial run at the end, so try looking ahead */
217 run_start = MASK_ALIGN - clz;
220 for (lookahead_idx = msk_idx + 1; lookahead_idx < msk->n_masks;
222 unsigned int s_idx, need;
223 lookahead_msk = msk->data[lookahead_idx];
225 /* if we're looking for free space, invert the mask */
227 lookahead_msk = ~lookahead_msk;
229 /* figure out how many consecutive bits we need here */
230 need = RTE_MIN(left, MASK_ALIGN);
232 for (s_idx = 0; s_idx < need - 1; s_idx++)
233 lookahead_msk &= lookahead_msk >> 1ULL;
235 /* if first bit is not set, we've lost the run */
236 if ((lookahead_msk & 1) == 0) {
238 * we've scanned this far, so we know there are
239 * no runs in the space we've lookahead-scanned
240 * as well, so skip that on next iteration.
242 ignore_msk = ~((1ULL << need) - 1);
243 msk_idx = lookahead_idx;
249 /* check if we've found what we were looking for */
256 /* we didn't find anything, so continue */
260 return MASK_GET_IDX(msk_idx, run_start);
262 /* we didn't find anything */
263 rte_errno = used ? ENOENT : ENOSPC;
268 find_next(const struct rte_fbarray *arr, unsigned int start, bool used)
270 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
272 unsigned int idx, first, first_mod;
273 unsigned int last, last_mod;
274 uint64_t last_msk, ignore_msk;
277 * mask only has granularity of MASK_ALIGN, but start may not be aligned
278 * on that boundary, so construct a special mask to exclude anything we
279 * don't want to see to avoid confusing ctz.
281 first = MASK_LEN_TO_IDX(start);
282 first_mod = MASK_LEN_TO_MOD(start);
283 ignore_msk = ~((1ULL << first_mod) - 1ULL);
285 /* array length may not be aligned, so calculate ignore mask for last
288 last = MASK_LEN_TO_IDX(arr->len);
289 last_mod = MASK_LEN_TO_MOD(arr->len);
290 last_msk = ~(-(1ULL) << last_mod);
292 for (idx = first; idx < msk->n_masks; idx++) {
293 uint64_t cur = msk->data[idx];
296 /* if we're looking for free entries, invert mask */
303 /* ignore everything before start on first iteration */
307 /* check if we have any entries */
312 * find first set bit - that will correspond to whatever it is
313 * that we're looking for.
315 found = __builtin_ctzll(cur);
316 return MASK_GET_IDX(idx, found);
318 /* we didn't find anything */
319 rte_errno = used ? ENOENT : ENOSPC;
324 find_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
326 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
328 unsigned int idx, first, first_mod;
329 unsigned int last, last_mod;
331 unsigned int need_len, result = 0;
333 /* array length may not be aligned, so calculate ignore mask for last
336 last = MASK_LEN_TO_IDX(arr->len);
337 last_mod = MASK_LEN_TO_MOD(arr->len);
338 last_msk = ~(-(1ULL) << last_mod);
340 first = MASK_LEN_TO_IDX(start);
341 first_mod = MASK_LEN_TO_MOD(start);
342 for (idx = first; idx < msk->n_masks; idx++, result += need_len) {
343 uint64_t cur = msk->data[idx];
344 unsigned int run_len;
346 need_len = MASK_ALIGN;
348 /* if we're looking for free entries, invert mask */
352 /* if this is last mask, ignore everything after last bit */
356 /* ignore everything before start on first iteration */
359 /* at the start, we don't need the full mask len */
360 need_len -= first_mod;
363 /* we will be looking for zeroes, so invert the mask */
366 /* if mask is zero, we have a complete run */
371 * see if current run ends before mask end.
373 run_len = __builtin_ctzll(cur);
375 /* add however many zeroes we've had in the last run and quit */
376 if (run_len < need_len) {
385 find_prev_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
388 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
390 unsigned int msk_idx, lookbehind_idx, first, first_mod;
394 * mask only has granularity of MASK_ALIGN, but start may not be aligned
395 * on that boundary, so construct a special mask to exclude anything we
396 * don't want to see to avoid confusing ctz.
398 first = MASK_LEN_TO_IDX(start);
399 first_mod = MASK_LEN_TO_MOD(start);
400 /* we're going backwards, so mask must start from the top */
401 ignore_msk = first_mod == MASK_ALIGN - 1 ?
402 -1ULL : /* prevent overflow */
403 ~(-1ULL << (first_mod + 1));
405 /* go backwards, include zero */
408 uint64_t cur_msk, lookbehind_msk;
409 unsigned int run_start, run_end, ctz, left;
412 * The process of getting n consecutive bits from the top for
413 * arbitrary n is a bit involved, but here it is in a nutshell:
415 * 1. let n be the number of consecutive bits we're looking for
416 * 2. check if n can fit in one mask, and if so, do n-1
417 * lshift-ands to see if there is an appropriate run inside
419 * 2a. if we found a run, bail out early
420 * 2b. if we didn't find a run, proceed
421 * 3. invert the mask and count trailing zeroes (that is, count
422 * how many consecutive set bits we had starting from the
423 * start of current mask) as k
424 * 3a. if k is 0, continue to next mask
425 * 3b. if k is not 0, we have a potential run
426 * 4. to satisfy our requirements, next mask must have n-k
427 * consecutive set bits at the end, so we will do (n-k-1)
428 * lshift-ands and check if last bit is set.
430 * Step 4 will need to be repeated if (n-k) > MASK_ALIGN until
431 * we either run out of masks, lose the run, or find what we
434 cur_msk = msk->data[msk_idx];
437 /* if we're looking for free spaces, invert the mask */
441 /* if we have an ignore mask, ignore once */
443 cur_msk &= ignore_msk;
447 /* if n can fit in within a single mask, do a search */
448 if (n <= MASK_ALIGN) {
449 uint64_t tmp_msk = cur_msk;
451 for (s_idx = 0; s_idx < n - 1; s_idx++)
452 tmp_msk &= tmp_msk << 1ULL;
453 /* we found what we were looking for */
455 /* clz will give us offset from end of mask, and
456 * we only get the end of our run, not start,
457 * so adjust result to point to where start
460 run_start = MASK_ALIGN -
461 __builtin_clzll(tmp_msk) - n;
462 return MASK_GET_IDX(msk_idx, run_start);
467 * we didn't find our run within the mask, or n > MASK_ALIGN,
468 * so we're going for plan B.
471 /* count trailing zeroes on inverted mask */
473 ctz = sizeof(cur_msk) * 8;
475 ctz = __builtin_ctzll(~cur_msk);
477 /* if there aren't any runs at the start either, just
483 /* we have a partial run at the start, so try looking behind */
484 run_end = MASK_GET_IDX(msk_idx, ctz);
487 /* go backwards, include zero */
488 lookbehind_idx = msk_idx - 1;
490 /* we can't lookbehind as we've run out of masks, so stop */
495 const uint64_t last_bit = 1ULL << (MASK_ALIGN - 1);
496 unsigned int s_idx, need;
498 lookbehind_msk = msk->data[lookbehind_idx];
500 /* if we're looking for free space, invert the mask */
502 lookbehind_msk = ~lookbehind_msk;
504 /* figure out how many consecutive bits we need here */
505 need = RTE_MIN(left, MASK_ALIGN);
507 for (s_idx = 0; s_idx < need - 1; s_idx++)
508 lookbehind_msk &= lookbehind_msk << 1ULL;
510 /* if last bit is not set, we've lost the run */
511 if ((lookbehind_msk & last_bit) == 0) {
513 * we've scanned this far, so we know there are
514 * no runs in the space we've lookbehind-scanned
515 * as well, so skip that on next iteration.
517 ignore_msk = -1ULL << need;
518 msk_idx = lookbehind_idx;
524 /* check if we've found what we were looking for */
529 } while ((lookbehind_idx--) != 0); /* decrement after check to
533 /* we didn't find anything, so continue */
537 /* we've found what we were looking for, but we only know where
538 * the run ended, so calculate start position.
541 } while (msk_idx-- != 0); /* decrement after check to include zero */
542 /* we didn't find anything */
543 rte_errno = used ? ENOENT : ENOSPC;
548 find_prev(const struct rte_fbarray *arr, unsigned int start, bool used)
550 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
552 unsigned int idx, first, first_mod;
556 * mask only has granularity of MASK_ALIGN, but start may not be aligned
557 * on that boundary, so construct a special mask to exclude anything we
558 * don't want to see to avoid confusing clz.
560 first = MASK_LEN_TO_IDX(start);
561 first_mod = MASK_LEN_TO_MOD(start);
562 /* we're going backwards, so mask must start from the top */
563 ignore_msk = first_mod == MASK_ALIGN - 1 ?
564 -1ULL : /* prevent overflow */
565 ~(-1ULL << (first_mod + 1));
567 /* go backwards, include zero */
570 uint64_t cur = msk->data[idx];
573 /* if we're looking for free entries, invert mask */
577 /* ignore everything before start on first iteration */
581 /* check if we have any entries */
586 * find last set bit - that will correspond to whatever it is
587 * that we're looking for. we're counting trailing zeroes, thus
588 * the value we get is counted from end of mask, so calculate
589 * position from start of mask.
591 found = MASK_ALIGN - __builtin_clzll(cur) - 1;
593 return MASK_GET_IDX(idx, found);
594 } while (idx-- != 0); /* decrement after check to include zero*/
596 /* we didn't find anything */
597 rte_errno = used ? ENOENT : ENOSPC;
602 find_rev_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
604 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
606 unsigned int idx, first, first_mod;
607 unsigned int need_len, result = 0;
609 first = MASK_LEN_TO_IDX(start);
610 first_mod = MASK_LEN_TO_MOD(start);
612 /* go backwards, include zero */
615 uint64_t cur = msk->data[idx];
616 unsigned int run_len;
618 need_len = MASK_ALIGN;
620 /* if we're looking for free entries, invert mask */
624 /* ignore everything after start on first iteration */
626 unsigned int end_len = MASK_ALIGN - first_mod - 1;
628 /* at the start, we don't need the full mask len */
632 /* we will be looking for zeroes, so invert the mask */
635 /* if mask is zero, we have a complete run */
640 * see where run ends, starting from the end.
642 run_len = __builtin_clzll(cur);
644 /* add however many zeroes we've had in the last run and quit */
645 if (run_len < need_len) {
651 } while (idx-- != 0); /* decrement after check to include zero */
656 set_used(struct rte_fbarray *arr, unsigned int idx, bool used)
658 struct used_mask *msk;
659 uint64_t msk_bit = 1ULL << MASK_LEN_TO_MOD(idx);
660 unsigned int msk_idx = MASK_LEN_TO_IDX(idx);
664 if (arr == NULL || idx >= arr->len) {
668 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
671 /* prevent array from changing under us */
672 rte_rwlock_write_lock(&arr->rwlock);
674 already_used = (msk->data[msk_idx] & msk_bit) != 0;
676 /* nothing to be done */
677 if (used == already_used)
681 msk->data[msk_idx] |= msk_bit;
684 msk->data[msk_idx] &= ~msk_bit;
688 rte_rwlock_write_unlock(&arr->rwlock);
694 fully_validate(const char *name, unsigned int elt_sz, unsigned int len)
696 if (name == NULL || elt_sz == 0 || len == 0 || len > INT_MAX) {
701 if (strnlen(name, RTE_FBARRAY_NAME_LEN) == RTE_FBARRAY_NAME_LEN) {
702 rte_errno = ENAMETOOLONG;
709 rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,
712 size_t page_sz, mmap_len;
714 struct used_mask *msk;
715 struct mem_area *ma = NULL;
718 const struct internal_config *internal_conf =
719 eal_get_internal_configuration();
726 if (fully_validate(name, elt_sz, len))
729 /* allocate mem area before doing anything */
730 ma = malloc(sizeof(*ma));
736 page_sz = rte_mem_page_size();
737 if (page_sz == (size_t)-1) {
742 /* calculate our memory limits */
743 mmap_len = calc_data_size(page_sz, elt_sz, len);
745 data = eal_get_virtual_area(NULL, &mmap_len, page_sz, 0, 0);
751 rte_spinlock_lock(&mem_area_lock);
755 if (internal_conf->no_shconf) {
756 /* remap virtual area as writable */
757 static const int flags = RTE_MAP_FORCE_ADDRESS |
758 RTE_MAP_PRIVATE | RTE_MAP_ANONYMOUS;
759 void *new_data = rte_mem_map(data, mmap_len,
760 RTE_PROT_READ | RTE_PROT_WRITE, flags, fd, 0);
761 if (new_data == NULL) {
762 RTE_LOG(DEBUG, EAL, "%s(): couldn't remap anonymous memory: %s\n",
763 __func__, rte_strerror(rte_errno));
767 eal_get_fbarray_path(path, sizeof(path), name);
770 * Each fbarray is unique to process namespace, i.e. the
771 * filename depends on process prefix. Try to take out a lock
772 * and see if we succeed. If we don't, someone else is using it
775 fd = eal_file_open(path, EAL_OPEN_CREATE | EAL_OPEN_READWRITE);
777 RTE_LOG(DEBUG, EAL, "%s(): couldn't open %s: %s\n",
778 __func__, path, rte_strerror(rte_errno));
780 } else if (eal_file_lock(
781 fd, EAL_FLOCK_EXCLUSIVE, EAL_FLOCK_RETURN)) {
782 RTE_LOG(DEBUG, EAL, "%s(): couldn't lock %s: %s\n",
783 __func__, path, rte_strerror(rte_errno));
788 /* take out a non-exclusive lock, so that other processes could
789 * still attach to it, but no other process could reinitialize
792 if (eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN))
795 if (resize_and_map(fd, data, mmap_len))
802 /* do not close fd - keep it until detach/destroy */
803 TAILQ_INSERT_TAIL(&mem_area_tailq, ma, next);
805 /* initialize the data */
806 memset(data, 0, mmap_len);
808 /* populate data structure */
809 strlcpy(arr->name, name, sizeof(arr->name));
812 arr->elt_sz = elt_sz;
815 msk = get_used_mask(data, elt_sz, len);
816 msk->n_masks = MASK_LEN_TO_IDX(RTE_ALIGN_CEIL(len, MASK_ALIGN));
818 rte_rwlock_init(&arr->rwlock);
820 rte_spinlock_unlock(&mem_area_lock);
825 rte_mem_unmap(data, mmap_len);
830 rte_spinlock_unlock(&mem_area_lock);
835 rte_fbarray_attach(struct rte_fbarray *arr)
837 struct mem_area *ma = NULL, *tmp = NULL;
838 size_t page_sz, mmap_len;
849 * we don't need to synchronize attach as two values we need (element
850 * size and array length) are constant for the duration of life of
851 * the array, so the parts we care about will not race.
854 if (fully_validate(arr->name, arr->elt_sz, arr->len))
857 ma = malloc(sizeof(*ma));
863 page_sz = rte_mem_page_size();
864 if (page_sz == (size_t)-1) {
869 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
871 /* check the tailq - maybe user has already mapped this address space */
872 rte_spinlock_lock(&mem_area_lock);
874 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
875 if (overlap(tmp, arr->data, mmap_len)) {
881 /* we know this memory area is unique, so proceed */
883 data = eal_get_virtual_area(arr->data, &mmap_len, page_sz, 0, 0);
887 eal_get_fbarray_path(path, sizeof(path), arr->name);
889 fd = eal_file_open(path, EAL_OPEN_READWRITE);
894 /* lock the file, to let others know we're using it */
895 if (eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN))
898 if (resize_and_map(fd, data, mmap_len))
901 /* store our new memory area */
903 ma->fd = fd; /* keep fd until detach/destroy */
906 TAILQ_INSERT_TAIL(&mem_area_tailq, ma, next);
910 rte_spinlock_unlock(&mem_area_lock);
914 rte_mem_unmap(data, mmap_len);
918 rte_spinlock_unlock(&mem_area_lock);
923 rte_fbarray_detach(struct rte_fbarray *arr)
925 struct mem_area *tmp = NULL;
935 * we don't need to synchronize detach as two values we need (element
936 * size and total capacity) are constant for the duration of life of
937 * the array, so the parts we care about will not race. if the user is
938 * detaching while doing something else in the same process, we can't
939 * really do anything about it, things will blow up either way.
942 size_t page_sz = rte_mem_page_size();
943 if (page_sz == (size_t)-1)
946 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
948 /* does this area exist? */
949 rte_spinlock_lock(&mem_area_lock);
951 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
952 if (tmp->addr == arr->data && tmp->len == mmap_len)
961 rte_mem_unmap(arr->data, mmap_len);
963 /* area is unmapped, close fd and remove the tailq entry */
966 TAILQ_REMOVE(&mem_area_tailq, tmp, next);
971 rte_spinlock_unlock(&mem_area_lock);
976 rte_fbarray_destroy(struct rte_fbarray *arr)
978 struct mem_area *tmp = NULL;
982 const struct internal_config *internal_conf =
983 eal_get_internal_configuration();
991 * we don't need to synchronize detach as two values we need (element
992 * size and total capacity) are constant for the duration of life of
993 * the array, so the parts we care about will not race. if the user is
994 * detaching while doing something else in the same process, we can't
995 * really do anything about it, things will blow up either way.
998 size_t page_sz = rte_mem_page_size();
999 if (page_sz == (size_t)-1)
1002 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
1004 /* does this area exist? */
1005 rte_spinlock_lock(&mem_area_lock);
1007 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
1008 if (tmp->addr == arr->data && tmp->len == mmap_len)
1016 /* with no shconf, there were never any files to begin with */
1017 if (!internal_conf->no_shconf) {
1019 * attempt to get an exclusive lock on the file, to ensure it
1020 * has been detached by all other processes
1023 if (eal_file_lock(fd, EAL_FLOCK_EXCLUSIVE, EAL_FLOCK_RETURN)) {
1024 RTE_LOG(DEBUG, EAL, "Cannot destroy fbarray - another process is using it\n");
1030 /* we're OK to destroy the file */
1031 eal_get_fbarray_path(path, sizeof(path), arr->name);
1033 RTE_LOG(DEBUG, EAL, "Cannot unlink fbarray: %s\n",
1037 * we're still holding an exclusive lock, so drop it to
1040 eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN);
1047 rte_mem_unmap(arr->data, mmap_len);
1049 /* area is unmapped, remove the tailq entry */
1050 TAILQ_REMOVE(&mem_area_tailq, tmp, next);
1054 /* reset the fbarray structure */
1055 memset(arr, 0, sizeof(*arr));
1057 rte_spinlock_unlock(&mem_area_lock);
1062 rte_fbarray_get(const struct rte_fbarray *arr, unsigned int idx)
1070 if (idx >= arr->len) {
1075 ret = RTE_PTR_ADD(arr->data, idx * arr->elt_sz);
1081 rte_fbarray_set_used(struct rte_fbarray *arr, unsigned int idx)
1083 return set_used(arr, idx, true);
1087 rte_fbarray_set_free(struct rte_fbarray *arr, unsigned int idx)
1089 return set_used(arr, idx, false);
1093 rte_fbarray_is_used(struct rte_fbarray *arr, unsigned int idx)
1095 struct used_mask *msk;
1100 if (arr == NULL || idx >= arr->len) {
1105 /* prevent array from changing under us */
1106 rte_rwlock_read_lock(&arr->rwlock);
1108 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
1109 msk_idx = MASK_LEN_TO_IDX(idx);
1110 msk_bit = 1ULL << MASK_LEN_TO_MOD(idx);
1112 ret = (msk->data[msk_idx] & msk_bit) != 0;
1114 rte_rwlock_read_unlock(&arr->rwlock);
1120 fbarray_find(struct rte_fbarray *arr, unsigned int start, bool next, bool used)
1124 if (arr == NULL || start >= arr->len) {
1129 /* prevent array from changing under us */
1130 rte_rwlock_read_lock(&arr->rwlock);
1132 /* cheap checks to prevent doing useless work */
1134 if (arr->len == arr->count) {
1138 if (arr->count == 0) {
1143 if (arr->count == 0) {
1147 if (arr->len == arr->count) {
1153 ret = find_next(arr, start, used);
1155 ret = find_prev(arr, start, used);
1157 rte_rwlock_read_unlock(&arr->rwlock);
1162 rte_fbarray_find_next_free(struct rte_fbarray *arr, unsigned int start)
1164 return fbarray_find(arr, start, true, false);
1168 rte_fbarray_find_next_used(struct rte_fbarray *arr, unsigned int start)
1170 return fbarray_find(arr, start, true, true);
1174 rte_fbarray_find_prev_free(struct rte_fbarray *arr, unsigned int start)
1176 return fbarray_find(arr, start, false, false);
1180 rte_fbarray_find_prev_used(struct rte_fbarray *arr, unsigned int start)
1182 return fbarray_find(arr, start, false, true);
1186 fbarray_find_n(struct rte_fbarray *arr, unsigned int start, unsigned int n,
1187 bool next, bool used)
1191 if (arr == NULL || start >= arr->len || n > arr->len || n == 0) {
1195 if (next && (arr->len - start) < n) {
1196 rte_errno = used ? ENOENT : ENOSPC;
1199 if (!next && start < (n - 1)) {
1200 rte_errno = used ? ENOENT : ENOSPC;
1204 /* prevent array from changing under us */
1205 rte_rwlock_read_lock(&arr->rwlock);
1207 /* cheap checks to prevent doing useless work */
1209 if (arr->len == arr->count || arr->len - arr->count < n) {
1213 if (arr->count == 0) {
1214 ret = next ? start : start - n + 1;
1218 if (arr->count < n) {
1222 if (arr->count == arr->len) {
1223 ret = next ? start : start - n + 1;
1229 ret = find_next_n(arr, start, n, used);
1231 ret = find_prev_n(arr, start, n, used);
1233 rte_rwlock_read_unlock(&arr->rwlock);
1238 rte_fbarray_find_next_n_free(struct rte_fbarray *arr, unsigned int start,
1241 return fbarray_find_n(arr, start, n, true, false);
1245 rte_fbarray_find_next_n_used(struct rte_fbarray *arr, unsigned int start,
1248 return fbarray_find_n(arr, start, n, true, true);
1252 rte_fbarray_find_prev_n_free(struct rte_fbarray *arr, unsigned int start,
1255 return fbarray_find_n(arr, start, n, false, false);
1259 rte_fbarray_find_prev_n_used(struct rte_fbarray *arr, unsigned int start,
1262 return fbarray_find_n(arr, start, n, false, true);
1266 fbarray_find_contig(struct rte_fbarray *arr, unsigned int start, bool next,
1271 if (arr == NULL || start >= arr->len) {
1276 /* prevent array from changing under us */
1277 rte_rwlock_read_lock(&arr->rwlock);
1279 /* cheap checks to prevent doing useless work */
1281 if (arr->count == 0) {
1285 if (next && arr->count == arr->len) {
1286 ret = arr->len - start;
1289 if (!next && arr->count == arr->len) {
1294 if (arr->len == arr->count) {
1298 if (next && arr->count == 0) {
1299 ret = arr->len - start;
1302 if (!next && arr->count == 0) {
1309 ret = find_contig(arr, start, used);
1311 ret = find_rev_contig(arr, start, used);
1313 rte_rwlock_read_unlock(&arr->rwlock);
1318 fbarray_find_biggest(struct rte_fbarray *arr, unsigned int start, bool used,
1321 int cur_idx, next_idx, cur_len, biggest_idx, biggest_len;
1322 /* don't stack if conditions, use function pointers instead */
1323 int (*find_func)(struct rte_fbarray *, unsigned int);
1324 int (*find_contig_func)(struct rte_fbarray *, unsigned int);
1326 if (arr == NULL || start >= arr->len) {
1330 /* the other API calls already do their fair share of cheap checks, so
1331 * no need to do them here.
1334 /* the API's called are thread-safe, but something may still happen
1335 * between the API calls, so lock the fbarray. all other API's are
1336 * read-locking the fbarray, so read lock here is OK.
1338 rte_rwlock_read_lock(&arr->rwlock);
1340 /* pick out appropriate functions */
1343 find_func = rte_fbarray_find_prev_used;
1344 find_contig_func = rte_fbarray_find_rev_contig_used;
1346 find_func = rte_fbarray_find_next_used;
1347 find_contig_func = rte_fbarray_find_contig_used;
1351 find_func = rte_fbarray_find_prev_free;
1352 find_contig_func = rte_fbarray_find_rev_contig_free;
1354 find_func = rte_fbarray_find_next_free;
1355 find_contig_func = rte_fbarray_find_contig_free;
1360 biggest_idx = -1; /* default is error */
1363 cur_idx = find_func(arr, cur_idx);
1365 /* block found, check its length */
1367 cur_len = find_contig_func(arr, cur_idx);
1368 /* decide where we go next */
1369 next_idx = rev ? cur_idx - cur_len : cur_idx + cur_len;
1370 /* move current index to start of chunk */
1371 cur_idx = rev ? next_idx + 1 : cur_idx;
1373 if (cur_len > biggest_len) {
1374 biggest_idx = cur_idx;
1375 biggest_len = cur_len;
1378 /* in reverse mode, next_idx may be -1 if chunk started
1379 * at array beginning. this means there's no more work
1385 /* nothing more to find, stop. however, a failed API
1386 * call has set rte_errno, which we want to ignore, as
1387 * reaching the end of fbarray is not an error.
1393 /* if we didn't find anything at all, set rte_errno */
1394 if (biggest_idx < 0)
1395 rte_errno = used ? ENOENT : ENOSPC;
1397 rte_rwlock_read_unlock(&arr->rwlock);
1402 rte_fbarray_find_biggest_free(struct rte_fbarray *arr, unsigned int start)
1404 return fbarray_find_biggest(arr, start, false, false);
1408 rte_fbarray_find_biggest_used(struct rte_fbarray *arr, unsigned int start)
1410 return fbarray_find_biggest(arr, start, true, false);
1414 rte_fbarray_find_rev_biggest_free(struct rte_fbarray *arr, unsigned int start)
1416 return fbarray_find_biggest(arr, start, false, true);
1420 rte_fbarray_find_rev_biggest_used(struct rte_fbarray *arr, unsigned int start)
1422 return fbarray_find_biggest(arr, start, true, true);
1427 rte_fbarray_find_contig_free(struct rte_fbarray *arr, unsigned int start)
1429 return fbarray_find_contig(arr, start, true, false);
1433 rte_fbarray_find_contig_used(struct rte_fbarray *arr, unsigned int start)
1435 return fbarray_find_contig(arr, start, true, true);
1439 rte_fbarray_find_rev_contig_free(struct rte_fbarray *arr, unsigned int start)
1441 return fbarray_find_contig(arr, start, false, false);
1445 rte_fbarray_find_rev_contig_used(struct rte_fbarray *arr, unsigned int start)
1447 return fbarray_find_contig(arr, start, false, true);
1451 rte_fbarray_find_idx(const struct rte_fbarray *arr, const void *elt)
1457 * no need to synchronize as it doesn't matter if underlying data
1458 * changes - we're doing pointer arithmetic here.
1461 if (arr == NULL || elt == NULL) {
1465 end = RTE_PTR_ADD(arr->data, arr->elt_sz * arr->len);
1466 if (elt < arr->data || elt >= end) {
1471 ret = RTE_PTR_DIFF(elt, arr->data) / arr->elt_sz;
1477 rte_fbarray_dump_metadata(struct rte_fbarray *arr, FILE *f)
1479 struct used_mask *msk;
1482 if (arr == NULL || f == NULL) {
1487 if (fully_validate(arr->name, arr->elt_sz, arr->len)) {
1488 fprintf(f, "Invalid file-backed array\n");
1492 /* prevent array from changing under us */
1493 rte_rwlock_read_lock(&arr->rwlock);
1495 fprintf(f, "File-backed array: %s\n", arr->name);
1496 fprintf(f, "size: %i occupied: %i elt_sz: %i\n",
1497 arr->len, arr->count, arr->elt_sz);
1499 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
1501 for (i = 0; i < msk->n_masks; i++)
1502 fprintf(f, "msk idx %i: 0x%016" PRIx64 "\n", i, msk->data[i]);
1504 rte_rwlock_read_unlock(&arr->rwlock);