1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
12 #include <rte_common.h>
13 #include <rte_eal_paging.h>
14 #include <rte_errno.h>
16 #include <rte_spinlock.h>
18 #include "eal_filesystem.h"
19 #include "eal_private.h"
21 #include "rte_fbarray.h"
23 #define MASK_SHIFT 6ULL
24 #define MASK_ALIGN (1ULL << MASK_SHIFT)
25 #define MASK_LEN_TO_IDX(x) ((x) >> MASK_SHIFT)
26 #define MASK_LEN_TO_MOD(x) ((x) - RTE_ALIGN_FLOOR(x, MASK_ALIGN))
27 #define MASK_GET_IDX(idx, mod) ((idx << MASK_SHIFT) + mod)
30 * We use this to keep track of created/attached memory areas to prevent user
31 * errors in API usage.
34 TAILQ_ENTRY(mem_area) next;
39 TAILQ_HEAD(mem_area_head, mem_area);
40 /* local per-process tailq */
41 static struct mem_area_head mem_area_tailq =
42 TAILQ_HEAD_INITIALIZER(mem_area_tailq);
43 static rte_spinlock_t mem_area_lock = RTE_SPINLOCK_INITIALIZER;
46 * This is a mask that is always stored at the end of array, to provide fast
47 * way of finding free/used spots without looping through each element.
56 calc_mask_size(unsigned int len)
58 /* mask must be multiple of MASK_ALIGN, even though length of array
59 * itself may not be aligned on that boundary.
61 len = RTE_ALIGN_CEIL(len, MASK_ALIGN);
62 return sizeof(struct used_mask) +
63 sizeof(uint64_t) * MASK_LEN_TO_IDX(len);
67 calc_data_size(size_t page_sz, unsigned int elt_sz, unsigned int len)
69 size_t data_sz = elt_sz * len;
70 size_t msk_sz = calc_mask_size(len);
71 return RTE_ALIGN_CEIL(data_sz + msk_sz, page_sz);
74 static struct used_mask *
75 get_used_mask(void *data, unsigned int elt_sz, unsigned int len)
77 return (struct used_mask *) RTE_PTR_ADD(data, elt_sz * len);
81 resize_and_map(int fd, const char *path, void *addr, size_t len)
85 if (eal_file_truncate(fd, len)) {
86 RTE_LOG(ERR, EAL, "Cannot truncate %s\n", path);
90 map_addr = rte_mem_map(addr, len, RTE_PROT_READ | RTE_PROT_WRITE,
91 RTE_MAP_SHARED | RTE_MAP_FORCE_ADDRESS, fd, 0);
92 if (map_addr != addr) {
99 overlap(const struct mem_area *ma, const void *start, size_t len)
101 const void *end = RTE_PTR_ADD(start, len);
102 const void *ma_start = ma->addr;
103 const void *ma_end = RTE_PTR_ADD(ma->addr, ma->len);
106 if (start >= ma_start && start < ma_end)
109 if (end > ma_start && end < ma_end)
115 find_next_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
118 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
120 unsigned int msk_idx, lookahead_idx, first, first_mod;
121 unsigned int last, last_mod;
122 uint64_t last_msk, ignore_msk;
125 * mask only has granularity of MASK_ALIGN, but start may not be aligned
126 * on that boundary, so construct a special mask to exclude anything we
127 * don't want to see to avoid confusing ctz.
129 first = MASK_LEN_TO_IDX(start);
130 first_mod = MASK_LEN_TO_MOD(start);
131 ignore_msk = ~((1ULL << first_mod) - 1);
133 /* array length may not be aligned, so calculate ignore mask for last
136 last = MASK_LEN_TO_IDX(arr->len);
137 last_mod = MASK_LEN_TO_MOD(arr->len);
138 last_msk = ~(UINT64_MAX << last_mod);
140 for (msk_idx = first; msk_idx < msk->n_masks; msk_idx++) {
141 uint64_t cur_msk, lookahead_msk;
142 unsigned int run_start, clz, left;
145 * The process of getting n consecutive bits for arbitrary n is
146 * a bit involved, but here it is in a nutshell:
148 * 1. let n be the number of consecutive bits we're looking for
149 * 2. check if n can fit in one mask, and if so, do n-1
150 * rshift-ands to see if there is an appropriate run inside
152 * 2a. if we found a run, bail out early
153 * 2b. if we didn't find a run, proceed
154 * 3. invert the mask and count leading zeroes (that is, count
155 * how many consecutive set bits we had starting from the
156 * end of current mask) as k
157 * 3a. if k is 0, continue to next mask
158 * 3b. if k is not 0, we have a potential run
159 * 4. to satisfy our requirements, next mask must have n-k
160 * consecutive set bits right at the start, so we will do
161 * (n-k-1) rshift-ands and check if first bit is set.
163 * Step 4 will need to be repeated if (n-k) > MASK_ALIGN until
164 * we either run out of masks, lose the run, or find what we
167 cur_msk = msk->data[msk_idx];
170 /* if we're looking for free spaces, invert the mask */
174 /* combine current ignore mask with last index ignore mask */
176 ignore_msk |= last_msk;
178 /* if we have an ignore mask, ignore once */
180 cur_msk &= ignore_msk;
184 /* if n can fit in within a single mask, do a search */
185 if (n <= MASK_ALIGN) {
186 uint64_t tmp_msk = cur_msk;
188 for (s_idx = 0; s_idx < n - 1; s_idx++)
189 tmp_msk &= tmp_msk >> 1ULL;
190 /* we found what we were looking for */
192 run_start = __builtin_ctzll(tmp_msk);
193 return MASK_GET_IDX(msk_idx, run_start);
198 * we didn't find our run within the mask, or n > MASK_ALIGN,
199 * so we're going for plan B.
202 /* count leading zeroes on inverted mask */
204 clz = sizeof(cur_msk) * 8;
206 clz = __builtin_clzll(~cur_msk);
208 /* if there aren't any runs at the end either, just continue */
212 /* we have a partial run at the end, so try looking ahead */
213 run_start = MASK_ALIGN - clz;
216 for (lookahead_idx = msk_idx + 1; lookahead_idx < msk->n_masks;
218 unsigned int s_idx, need;
219 lookahead_msk = msk->data[lookahead_idx];
221 /* if we're looking for free space, invert the mask */
223 lookahead_msk = ~lookahead_msk;
225 /* figure out how many consecutive bits we need here */
226 need = RTE_MIN(left, MASK_ALIGN);
228 for (s_idx = 0; s_idx < need - 1; s_idx++)
229 lookahead_msk &= lookahead_msk >> 1ULL;
231 /* if first bit is not set, we've lost the run */
232 if ((lookahead_msk & 1) == 0) {
234 * we've scanned this far, so we know there are
235 * no runs in the space we've lookahead-scanned
236 * as well, so skip that on next iteration.
238 ignore_msk = ~((1ULL << need) - 1);
239 msk_idx = lookahead_idx;
245 /* check if we've found what we were looking for */
252 /* we didn't find anything, so continue */
256 return MASK_GET_IDX(msk_idx, run_start);
258 /* we didn't find anything */
259 rte_errno = used ? ENOENT : ENOSPC;
264 find_next(const struct rte_fbarray *arr, unsigned int start, bool used)
266 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
268 unsigned int idx, first, first_mod;
269 unsigned int last, last_mod;
270 uint64_t last_msk, ignore_msk;
273 * mask only has granularity of MASK_ALIGN, but start may not be aligned
274 * on that boundary, so construct a special mask to exclude anything we
275 * don't want to see to avoid confusing ctz.
277 first = MASK_LEN_TO_IDX(start);
278 first_mod = MASK_LEN_TO_MOD(start);
279 ignore_msk = ~((1ULL << first_mod) - 1ULL);
281 /* array length may not be aligned, so calculate ignore mask for last
284 last = MASK_LEN_TO_IDX(arr->len);
285 last_mod = MASK_LEN_TO_MOD(arr->len);
286 last_msk = ~(-(1ULL) << last_mod);
288 for (idx = first; idx < msk->n_masks; idx++) {
289 uint64_t cur = msk->data[idx];
292 /* if we're looking for free entries, invert mask */
299 /* ignore everything before start on first iteration */
303 /* check if we have any entries */
308 * find first set bit - that will correspond to whatever it is
309 * that we're looking for.
311 found = __builtin_ctzll(cur);
312 return MASK_GET_IDX(idx, found);
314 /* we didn't find anything */
315 rte_errno = used ? ENOENT : ENOSPC;
320 find_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
322 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
324 unsigned int idx, first, first_mod;
325 unsigned int last, last_mod;
327 unsigned int need_len, result = 0;
329 /* array length may not be aligned, so calculate ignore mask for last
332 last = MASK_LEN_TO_IDX(arr->len);
333 last_mod = MASK_LEN_TO_MOD(arr->len);
334 last_msk = ~(-(1ULL) << last_mod);
336 first = MASK_LEN_TO_IDX(start);
337 first_mod = MASK_LEN_TO_MOD(start);
338 for (idx = first; idx < msk->n_masks; idx++, result += need_len) {
339 uint64_t cur = msk->data[idx];
340 unsigned int run_len;
342 need_len = MASK_ALIGN;
344 /* if we're looking for free entries, invert mask */
348 /* if this is last mask, ignore everything after last bit */
352 /* ignore everything before start on first iteration */
355 /* at the start, we don't need the full mask len */
356 need_len -= first_mod;
359 /* we will be looking for zeroes, so invert the mask */
362 /* if mask is zero, we have a complete run */
367 * see if current run ends before mask end.
369 run_len = __builtin_ctzll(cur);
371 /* add however many zeroes we've had in the last run and quit */
372 if (run_len < need_len) {
381 find_prev_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
384 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
386 unsigned int msk_idx, lookbehind_idx, first, first_mod;
390 * mask only has granularity of MASK_ALIGN, but start may not be aligned
391 * on that boundary, so construct a special mask to exclude anything we
392 * don't want to see to avoid confusing ctz.
394 first = MASK_LEN_TO_IDX(start);
395 first_mod = MASK_LEN_TO_MOD(start);
396 /* we're going backwards, so mask must start from the top */
397 ignore_msk = first_mod == MASK_ALIGN - 1 ?
398 UINT64_MAX : /* prevent overflow */
399 ~(UINT64_MAX << (first_mod + 1));
401 /* go backwards, include zero */
404 uint64_t cur_msk, lookbehind_msk;
405 unsigned int run_start, run_end, ctz, left;
408 * The process of getting n consecutive bits from the top for
409 * arbitrary n is a bit involved, but here it is in a nutshell:
411 * 1. let n be the number of consecutive bits we're looking for
412 * 2. check if n can fit in one mask, and if so, do n-1
413 * lshift-ands to see if there is an appropriate run inside
415 * 2a. if we found a run, bail out early
416 * 2b. if we didn't find a run, proceed
417 * 3. invert the mask and count trailing zeroes (that is, count
418 * how many consecutive set bits we had starting from the
419 * start of current mask) as k
420 * 3a. if k is 0, continue to next mask
421 * 3b. if k is not 0, we have a potential run
422 * 4. to satisfy our requirements, next mask must have n-k
423 * consecutive set bits at the end, so we will do (n-k-1)
424 * lshift-ands and check if last bit is set.
426 * Step 4 will need to be repeated if (n-k) > MASK_ALIGN until
427 * we either run out of masks, lose the run, or find what we
430 cur_msk = msk->data[msk_idx];
433 /* if we're looking for free spaces, invert the mask */
437 /* if we have an ignore mask, ignore once */
439 cur_msk &= ignore_msk;
443 /* if n can fit in within a single mask, do a search */
444 if (n <= MASK_ALIGN) {
445 uint64_t tmp_msk = cur_msk;
447 for (s_idx = 0; s_idx < n - 1; s_idx++)
448 tmp_msk &= tmp_msk << 1ULL;
449 /* we found what we were looking for */
451 /* clz will give us offset from end of mask, and
452 * we only get the end of our run, not start,
453 * so adjust result to point to where start
456 run_start = MASK_ALIGN -
457 __builtin_clzll(tmp_msk) - n;
458 return MASK_GET_IDX(msk_idx, run_start);
463 * we didn't find our run within the mask, or n > MASK_ALIGN,
464 * so we're going for plan B.
467 /* count trailing zeroes on inverted mask */
469 ctz = sizeof(cur_msk) * 8;
471 ctz = __builtin_ctzll(~cur_msk);
473 /* if there aren't any runs at the start either, just
479 /* we have a partial run at the start, so try looking behind */
480 run_end = MASK_GET_IDX(msk_idx, ctz);
483 /* go backwards, include zero */
484 lookbehind_idx = msk_idx - 1;
486 /* we can't lookbehind as we've run out of masks, so stop */
491 const uint64_t last_bit = 1ULL << (MASK_ALIGN - 1);
492 unsigned int s_idx, need;
494 lookbehind_msk = msk->data[lookbehind_idx];
496 /* if we're looking for free space, invert the mask */
498 lookbehind_msk = ~lookbehind_msk;
500 /* figure out how many consecutive bits we need here */
501 need = RTE_MIN(left, MASK_ALIGN);
503 for (s_idx = 0; s_idx < need - 1; s_idx++)
504 lookbehind_msk &= lookbehind_msk << 1ULL;
506 /* if last bit is not set, we've lost the run */
507 if ((lookbehind_msk & last_bit) == 0) {
509 * we've scanned this far, so we know there are
510 * no runs in the space we've lookbehind-scanned
511 * as well, so skip that on next iteration.
513 ignore_msk = UINT64_MAX << need;
514 msk_idx = lookbehind_idx;
520 /* check if we've found what we were looking for */
525 } while ((lookbehind_idx--) != 0); /* decrement after check to
529 /* we didn't find anything, so continue */
533 /* we've found what we were looking for, but we only know where
534 * the run ended, so calculate start position.
537 } while (msk_idx-- != 0); /* decrement after check to include zero */
538 /* we didn't find anything */
539 rte_errno = used ? ENOENT : ENOSPC;
544 find_prev(const struct rte_fbarray *arr, unsigned int start, bool used)
546 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
548 unsigned int idx, first, first_mod;
552 * mask only has granularity of MASK_ALIGN, but start may not be aligned
553 * on that boundary, so construct a special mask to exclude anything we
554 * don't want to see to avoid confusing clz.
556 first = MASK_LEN_TO_IDX(start);
557 first_mod = MASK_LEN_TO_MOD(start);
558 /* we're going backwards, so mask must start from the top */
559 ignore_msk = first_mod == MASK_ALIGN - 1 ?
560 UINT64_MAX : /* prevent overflow */
561 ~(UINT64_MAX << (first_mod + 1));
563 /* go backwards, include zero */
566 uint64_t cur = msk->data[idx];
569 /* if we're looking for free entries, invert mask */
573 /* ignore everything before start on first iteration */
577 /* check if we have any entries */
582 * find last set bit - that will correspond to whatever it is
583 * that we're looking for. we're counting trailing zeroes, thus
584 * the value we get is counted from end of mask, so calculate
585 * position from start of mask.
587 found = MASK_ALIGN - __builtin_clzll(cur) - 1;
589 return MASK_GET_IDX(idx, found);
590 } while (idx-- != 0); /* decrement after check to include zero*/
592 /* we didn't find anything */
593 rte_errno = used ? ENOENT : ENOSPC;
598 find_rev_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
600 const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
602 unsigned int idx, first, first_mod;
603 unsigned int need_len, result = 0;
605 first = MASK_LEN_TO_IDX(start);
606 first_mod = MASK_LEN_TO_MOD(start);
608 /* go backwards, include zero */
611 uint64_t cur = msk->data[idx];
612 unsigned int run_len;
614 need_len = MASK_ALIGN;
616 /* if we're looking for free entries, invert mask */
620 /* ignore everything after start on first iteration */
622 unsigned int end_len = MASK_ALIGN - first_mod - 1;
624 /* at the start, we don't need the full mask len */
628 /* we will be looking for zeroes, so invert the mask */
631 /* if mask is zero, we have a complete run */
636 * see where run ends, starting from the end.
638 run_len = __builtin_clzll(cur);
640 /* add however many zeroes we've had in the last run and quit */
641 if (run_len < need_len) {
647 } while (idx-- != 0); /* decrement after check to include zero */
652 set_used(struct rte_fbarray *arr, unsigned int idx, bool used)
654 struct used_mask *msk;
655 uint64_t msk_bit = 1ULL << MASK_LEN_TO_MOD(idx);
656 unsigned int msk_idx = MASK_LEN_TO_IDX(idx);
660 if (arr == NULL || idx >= arr->len) {
664 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
667 /* prevent array from changing under us */
668 rte_rwlock_write_lock(&arr->rwlock);
670 already_used = (msk->data[msk_idx] & msk_bit) != 0;
672 /* nothing to be done */
673 if (used == already_used)
677 msk->data[msk_idx] |= msk_bit;
680 msk->data[msk_idx] &= ~msk_bit;
684 rte_rwlock_write_unlock(&arr->rwlock);
690 fully_validate(const char *name, unsigned int elt_sz, unsigned int len)
692 if (name == NULL || elt_sz == 0 || len == 0 || len > INT_MAX) {
697 if (strnlen(name, RTE_FBARRAY_NAME_LEN) == RTE_FBARRAY_NAME_LEN) {
698 rte_errno = ENAMETOOLONG;
705 rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,
708 size_t page_sz, mmap_len;
710 struct used_mask *msk;
711 struct mem_area *ma = NULL;
714 const struct internal_config *internal_conf =
715 eal_get_internal_configuration();
722 if (fully_validate(name, elt_sz, len))
725 /* allocate mem area before doing anything */
726 ma = malloc(sizeof(*ma));
732 page_sz = rte_mem_page_size();
733 if (page_sz == (size_t)-1) {
738 /* calculate our memory limits */
739 mmap_len = calc_data_size(page_sz, elt_sz, len);
741 data = eal_get_virtual_area(NULL, &mmap_len, page_sz, 0, 0);
747 rte_spinlock_lock(&mem_area_lock);
751 if (internal_conf->no_shconf) {
752 /* remap virtual area as writable */
753 static const int flags = RTE_MAP_FORCE_ADDRESS |
754 RTE_MAP_PRIVATE | RTE_MAP_ANONYMOUS;
755 void *new_data = rte_mem_map(data, mmap_len,
756 RTE_PROT_READ | RTE_PROT_WRITE, flags, fd, 0);
757 if (new_data == NULL) {
758 RTE_LOG(DEBUG, EAL, "%s(): couldn't remap anonymous memory: %s\n",
759 __func__, rte_strerror(rte_errno));
763 eal_get_fbarray_path(path, sizeof(path), name);
766 * Each fbarray is unique to process namespace, i.e. the
767 * filename depends on process prefix. Try to take out a lock
768 * and see if we succeed. If we don't, someone else is using it
771 fd = eal_file_open(path, EAL_OPEN_CREATE | EAL_OPEN_READWRITE);
773 RTE_LOG(DEBUG, EAL, "%s(): couldn't open %s: %s\n",
774 __func__, path, rte_strerror(rte_errno));
776 } else if (eal_file_lock(
777 fd, EAL_FLOCK_EXCLUSIVE, EAL_FLOCK_RETURN)) {
778 RTE_LOG(DEBUG, EAL, "%s(): couldn't lock %s: %s\n",
779 __func__, path, rte_strerror(rte_errno));
784 /* take out a non-exclusive lock, so that other processes could
785 * still attach to it, but no other process could reinitialize
788 if (eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN))
791 if (resize_and_map(fd, path, data, mmap_len))
798 /* do not close fd - keep it until detach/destroy */
799 TAILQ_INSERT_TAIL(&mem_area_tailq, ma, next);
801 /* initialize the data */
802 memset(data, 0, mmap_len);
804 /* populate data structure */
805 strlcpy(arr->name, name, sizeof(arr->name));
808 arr->elt_sz = elt_sz;
811 msk = get_used_mask(data, elt_sz, len);
812 msk->n_masks = MASK_LEN_TO_IDX(RTE_ALIGN_CEIL(len, MASK_ALIGN));
814 rte_rwlock_init(&arr->rwlock);
816 rte_spinlock_unlock(&mem_area_lock);
821 rte_mem_unmap(data, mmap_len);
826 rte_spinlock_unlock(&mem_area_lock);
831 rte_fbarray_attach(struct rte_fbarray *arr)
833 struct mem_area *ma = NULL, *tmp = NULL;
834 size_t page_sz, mmap_len;
845 * we don't need to synchronize attach as two values we need (element
846 * size and array length) are constant for the duration of life of
847 * the array, so the parts we care about will not race.
850 if (fully_validate(arr->name, arr->elt_sz, arr->len))
853 ma = malloc(sizeof(*ma));
859 page_sz = rte_mem_page_size();
860 if (page_sz == (size_t)-1) {
865 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
867 /* check the tailq - maybe user has already mapped this address space */
868 rte_spinlock_lock(&mem_area_lock);
870 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
871 if (overlap(tmp, arr->data, mmap_len)) {
877 /* we know this memory area is unique, so proceed */
879 data = eal_get_virtual_area(arr->data, &mmap_len, page_sz, 0, 0);
883 eal_get_fbarray_path(path, sizeof(path), arr->name);
885 fd = eal_file_open(path, EAL_OPEN_READWRITE);
890 /* lock the file, to let others know we're using it */
891 if (eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN))
894 if (resize_and_map(fd, path, data, mmap_len))
897 /* store our new memory area */
899 ma->fd = fd; /* keep fd until detach/destroy */
902 TAILQ_INSERT_TAIL(&mem_area_tailq, ma, next);
906 rte_spinlock_unlock(&mem_area_lock);
910 rte_mem_unmap(data, mmap_len);
914 rte_spinlock_unlock(&mem_area_lock);
919 rte_fbarray_detach(struct rte_fbarray *arr)
921 struct mem_area *tmp = NULL;
931 * we don't need to synchronize detach as two values we need (element
932 * size and total capacity) are constant for the duration of life of
933 * the array, so the parts we care about will not race. if the user is
934 * detaching while doing something else in the same process, we can't
935 * really do anything about it, things will blow up either way.
938 size_t page_sz = rte_mem_page_size();
939 if (page_sz == (size_t)-1)
942 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
944 /* does this area exist? */
945 rte_spinlock_lock(&mem_area_lock);
947 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
948 if (tmp->addr == arr->data && tmp->len == mmap_len)
957 rte_mem_unmap(arr->data, mmap_len);
959 /* area is unmapped, close fd and remove the tailq entry */
962 TAILQ_REMOVE(&mem_area_tailq, tmp, next);
967 rte_spinlock_unlock(&mem_area_lock);
972 rte_fbarray_destroy(struct rte_fbarray *arr)
974 struct mem_area *tmp = NULL;
978 const struct internal_config *internal_conf =
979 eal_get_internal_configuration();
987 * we don't need to synchronize detach as two values we need (element
988 * size and total capacity) are constant for the duration of life of
989 * the array, so the parts we care about will not race. if the user is
990 * detaching while doing something else in the same process, we can't
991 * really do anything about it, things will blow up either way.
994 size_t page_sz = rte_mem_page_size();
995 if (page_sz == (size_t)-1)
998 mmap_len = calc_data_size(page_sz, arr->elt_sz, arr->len);
1000 /* does this area exist? */
1001 rte_spinlock_lock(&mem_area_lock);
1003 TAILQ_FOREACH(tmp, &mem_area_tailq, next) {
1004 if (tmp->addr == arr->data && tmp->len == mmap_len)
1012 /* with no shconf, there were never any files to begin with */
1013 if (!internal_conf->no_shconf) {
1015 * attempt to get an exclusive lock on the file, to ensure it
1016 * has been detached by all other processes
1019 if (eal_file_lock(fd, EAL_FLOCK_EXCLUSIVE, EAL_FLOCK_RETURN)) {
1020 RTE_LOG(DEBUG, EAL, "Cannot destroy fbarray - another process is using it\n");
1026 /* we're OK to destroy the file */
1027 eal_get_fbarray_path(path, sizeof(path), arr->name);
1029 RTE_LOG(DEBUG, EAL, "Cannot unlink fbarray: %s\n",
1033 * we're still holding an exclusive lock, so drop it to
1036 eal_file_lock(fd, EAL_FLOCK_SHARED, EAL_FLOCK_RETURN);
1043 rte_mem_unmap(arr->data, mmap_len);
1045 /* area is unmapped, remove the tailq entry */
1046 TAILQ_REMOVE(&mem_area_tailq, tmp, next);
1050 /* reset the fbarray structure */
1051 memset(arr, 0, sizeof(*arr));
1053 rte_spinlock_unlock(&mem_area_lock);
1058 rte_fbarray_get(const struct rte_fbarray *arr, unsigned int idx)
1066 if (idx >= arr->len) {
1071 ret = RTE_PTR_ADD(arr->data, idx * arr->elt_sz);
1077 rte_fbarray_set_used(struct rte_fbarray *arr, unsigned int idx)
1079 return set_used(arr, idx, true);
1083 rte_fbarray_set_free(struct rte_fbarray *arr, unsigned int idx)
1085 return set_used(arr, idx, false);
1089 rte_fbarray_is_used(struct rte_fbarray *arr, unsigned int idx)
1091 struct used_mask *msk;
1096 if (arr == NULL || idx >= arr->len) {
1101 /* prevent array from changing under us */
1102 rte_rwlock_read_lock(&arr->rwlock);
1104 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
1105 msk_idx = MASK_LEN_TO_IDX(idx);
1106 msk_bit = 1ULL << MASK_LEN_TO_MOD(idx);
1108 ret = (msk->data[msk_idx] & msk_bit) != 0;
1110 rte_rwlock_read_unlock(&arr->rwlock);
1116 fbarray_find(struct rte_fbarray *arr, unsigned int start, bool next, bool used)
1120 if (arr == NULL || start >= arr->len) {
1125 /* prevent array from changing under us */
1126 rte_rwlock_read_lock(&arr->rwlock);
1128 /* cheap checks to prevent doing useless work */
1130 if (arr->len == arr->count) {
1134 if (arr->count == 0) {
1139 if (arr->count == 0) {
1143 if (arr->len == arr->count) {
1149 ret = find_next(arr, start, used);
1151 ret = find_prev(arr, start, used);
1153 rte_rwlock_read_unlock(&arr->rwlock);
1158 rte_fbarray_find_next_free(struct rte_fbarray *arr, unsigned int start)
1160 return fbarray_find(arr, start, true, false);
1164 rte_fbarray_find_next_used(struct rte_fbarray *arr, unsigned int start)
1166 return fbarray_find(arr, start, true, true);
1170 rte_fbarray_find_prev_free(struct rte_fbarray *arr, unsigned int start)
1172 return fbarray_find(arr, start, false, false);
1176 rte_fbarray_find_prev_used(struct rte_fbarray *arr, unsigned int start)
1178 return fbarray_find(arr, start, false, true);
1182 fbarray_find_n(struct rte_fbarray *arr, unsigned int start, unsigned int n,
1183 bool next, bool used)
1187 if (arr == NULL || start >= arr->len || n > arr->len || n == 0) {
1191 if (next && (arr->len - start) < n) {
1192 rte_errno = used ? ENOENT : ENOSPC;
1195 if (!next && start < (n - 1)) {
1196 rte_errno = used ? ENOENT : ENOSPC;
1200 /* prevent array from changing under us */
1201 rte_rwlock_read_lock(&arr->rwlock);
1203 /* cheap checks to prevent doing useless work */
1205 if (arr->len == arr->count || arr->len - arr->count < n) {
1209 if (arr->count == 0) {
1210 ret = next ? start : start - n + 1;
1214 if (arr->count < n) {
1218 if (arr->count == arr->len) {
1219 ret = next ? start : start - n + 1;
1225 ret = find_next_n(arr, start, n, used);
1227 ret = find_prev_n(arr, start, n, used);
1229 rte_rwlock_read_unlock(&arr->rwlock);
1234 rte_fbarray_find_next_n_free(struct rte_fbarray *arr, unsigned int start,
1237 return fbarray_find_n(arr, start, n, true, false);
1241 rte_fbarray_find_next_n_used(struct rte_fbarray *arr, unsigned int start,
1244 return fbarray_find_n(arr, start, n, true, true);
1248 rte_fbarray_find_prev_n_free(struct rte_fbarray *arr, unsigned int start,
1251 return fbarray_find_n(arr, start, n, false, false);
1255 rte_fbarray_find_prev_n_used(struct rte_fbarray *arr, unsigned int start,
1258 return fbarray_find_n(arr, start, n, false, true);
1262 fbarray_find_contig(struct rte_fbarray *arr, unsigned int start, bool next,
1267 if (arr == NULL || start >= arr->len) {
1272 /* prevent array from changing under us */
1273 rte_rwlock_read_lock(&arr->rwlock);
1275 /* cheap checks to prevent doing useless work */
1277 if (arr->count == 0) {
1281 if (next && arr->count == arr->len) {
1282 ret = arr->len - start;
1285 if (!next && arr->count == arr->len) {
1290 if (arr->len == arr->count) {
1294 if (next && arr->count == 0) {
1295 ret = arr->len - start;
1298 if (!next && arr->count == 0) {
1305 ret = find_contig(arr, start, used);
1307 ret = find_rev_contig(arr, start, used);
1309 rte_rwlock_read_unlock(&arr->rwlock);
1314 fbarray_find_biggest(struct rte_fbarray *arr, unsigned int start, bool used,
1317 int cur_idx, next_idx, cur_len, biggest_idx, biggest_len;
1318 /* don't stack if conditions, use function pointers instead */
1319 int (*find_func)(struct rte_fbarray *, unsigned int);
1320 int (*find_contig_func)(struct rte_fbarray *, unsigned int);
1322 if (arr == NULL || start >= arr->len) {
1326 /* the other API calls already do their fair share of cheap checks, so
1327 * no need to do them here.
1330 /* the API's called are thread-safe, but something may still happen
1331 * between the API calls, so lock the fbarray. all other API's are
1332 * read-locking the fbarray, so read lock here is OK.
1334 rte_rwlock_read_lock(&arr->rwlock);
1336 /* pick out appropriate functions */
1339 find_func = rte_fbarray_find_prev_used;
1340 find_contig_func = rte_fbarray_find_rev_contig_used;
1342 find_func = rte_fbarray_find_next_used;
1343 find_contig_func = rte_fbarray_find_contig_used;
1347 find_func = rte_fbarray_find_prev_free;
1348 find_contig_func = rte_fbarray_find_rev_contig_free;
1350 find_func = rte_fbarray_find_next_free;
1351 find_contig_func = rte_fbarray_find_contig_free;
1356 biggest_idx = -1; /* default is error */
1359 cur_idx = find_func(arr, cur_idx);
1361 /* block found, check its length */
1363 cur_len = find_contig_func(arr, cur_idx);
1364 /* decide where we go next */
1365 next_idx = rev ? cur_idx - cur_len : cur_idx + cur_len;
1366 /* move current index to start of chunk */
1367 cur_idx = rev ? next_idx + 1 : cur_idx;
1369 if (cur_len > biggest_len) {
1370 biggest_idx = cur_idx;
1371 biggest_len = cur_len;
1374 /* in reverse mode, next_idx may be -1 if chunk started
1375 * at array beginning. this means there's no more work
1381 /* nothing more to find, stop. however, a failed API
1382 * call has set rte_errno, which we want to ignore, as
1383 * reaching the end of fbarray is not an error.
1389 /* if we didn't find anything at all, set rte_errno */
1390 if (biggest_idx < 0)
1391 rte_errno = used ? ENOENT : ENOSPC;
1393 rte_rwlock_read_unlock(&arr->rwlock);
1398 rte_fbarray_find_biggest_free(struct rte_fbarray *arr, unsigned int start)
1400 return fbarray_find_biggest(arr, start, false, false);
1404 rte_fbarray_find_biggest_used(struct rte_fbarray *arr, unsigned int start)
1406 return fbarray_find_biggest(arr, start, true, false);
1410 rte_fbarray_find_rev_biggest_free(struct rte_fbarray *arr, unsigned int start)
1412 return fbarray_find_biggest(arr, start, false, true);
1416 rte_fbarray_find_rev_biggest_used(struct rte_fbarray *arr, unsigned int start)
1418 return fbarray_find_biggest(arr, start, true, true);
1423 rte_fbarray_find_contig_free(struct rte_fbarray *arr, unsigned int start)
1425 return fbarray_find_contig(arr, start, true, false);
1429 rte_fbarray_find_contig_used(struct rte_fbarray *arr, unsigned int start)
1431 return fbarray_find_contig(arr, start, true, true);
1435 rte_fbarray_find_rev_contig_free(struct rte_fbarray *arr, unsigned int start)
1437 return fbarray_find_contig(arr, start, false, false);
1441 rte_fbarray_find_rev_contig_used(struct rte_fbarray *arr, unsigned int start)
1443 return fbarray_find_contig(arr, start, false, true);
1447 rte_fbarray_find_idx(const struct rte_fbarray *arr, const void *elt)
1453 * no need to synchronize as it doesn't matter if underlying data
1454 * changes - we're doing pointer arithmetic here.
1457 if (arr == NULL || elt == NULL) {
1461 end = RTE_PTR_ADD(arr->data, arr->elt_sz * arr->len);
1462 if (elt < arr->data || elt >= end) {
1467 ret = RTE_PTR_DIFF(elt, arr->data) / arr->elt_sz;
1473 rte_fbarray_dump_metadata(struct rte_fbarray *arr, FILE *f)
1475 struct used_mask *msk;
1478 if (arr == NULL || f == NULL) {
1483 if (fully_validate(arr->name, arr->elt_sz, arr->len)) {
1484 fprintf(f, "Invalid file-backed array\n");
1488 /* prevent array from changing under us */
1489 rte_rwlock_read_lock(&arr->rwlock);
1491 fprintf(f, "File-backed array: %s\n", arr->name);
1492 fprintf(f, "size: %i occupied: %i elt_sz: %i\n",
1493 arr->len, arr->count, arr->elt_sz);
1495 msk = get_used_mask(arr->data, arr->elt_sz, arr->len);
1497 for (i = 0; i < msk->n_masks; i++)
1498 fprintf(f, "msk idx %i: 0x%016" PRIx64 "\n", i, msk->data[i]);
1500 rte_rwlock_read_unlock(&arr->rwlock);