+
+static int
+rte_eal_memdevice_init(void)
+{
+ struct rte_config *config;
+ const struct internal_config *internal_conf;
+
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+ return 0;
+
+ internal_conf = eal_get_internal_configuration();
+ config = rte_eal_get_configuration();
+ config->mem_config->nchannel = internal_conf->force_nchannel;
+ config->mem_config->nrank = internal_conf->force_nrank;
+
+ return 0;
+}
+
+/* Lock page in physical memory and prevent from swapping. */
+int
+rte_mem_lock_page(const void *virt)
+{
+ uintptr_t virtual = (uintptr_t)virt;
+ size_t page_size = rte_mem_page_size();
+ uintptr_t aligned = RTE_PTR_ALIGN_FLOOR(virtual, page_size);
+ return rte_mem_lock((void *)aligned, page_size);
+}
+
+int
+rte_memseg_contig_walk_thread_unsafe(rte_memseg_contig_walk_t func, void *arg)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ int i, ms_idx, ret = 0;
+
+ for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+ struct rte_memseg_list *msl = &mcfg->memsegs[i];
+ const struct rte_memseg *ms;
+ struct rte_fbarray *arr;
+
+ if (msl->memseg_arr.count == 0)
+ continue;
+
+ arr = &msl->memseg_arr;
+
+ ms_idx = rte_fbarray_find_next_used(arr, 0);
+ while (ms_idx >= 0) {
+ int n_segs;
+ size_t len;
+
+ ms = rte_fbarray_get(arr, ms_idx);
+
+ /* find how many more segments there are, starting with
+ * this one.
+ */
+ n_segs = rte_fbarray_find_contig_used(arr, ms_idx);
+ len = n_segs * msl->page_sz;
+
+ ret = func(msl, ms, len, arg);
+ if (ret)
+ return ret;
+ ms_idx = rte_fbarray_find_next_used(arr,
+ ms_idx + n_segs);
+ }
+ }
+ return 0;
+}
+
+int
+rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg)
+{
+ int ret = 0;
+
+ /* do not allow allocations/frees/init while we iterate */
+ rte_mcfg_mem_read_lock();
+ ret = rte_memseg_contig_walk_thread_unsafe(func, arg);
+ rte_mcfg_mem_read_unlock();
+
+ return ret;
+}
+
+int
+rte_memseg_walk_thread_unsafe(rte_memseg_walk_t func, void *arg)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ int i, ms_idx, ret = 0;
+
+ for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+ struct rte_memseg_list *msl = &mcfg->memsegs[i];
+ const struct rte_memseg *ms;
+ struct rte_fbarray *arr;
+
+ if (msl->memseg_arr.count == 0)
+ continue;
+
+ arr = &msl->memseg_arr;
+
+ ms_idx = rte_fbarray_find_next_used(arr, 0);
+ while (ms_idx >= 0) {
+ ms = rte_fbarray_get(arr, ms_idx);
+ ret = func(msl, ms, arg);
+ if (ret)
+ return ret;
+ ms_idx = rte_fbarray_find_next_used(arr, ms_idx + 1);
+ }
+ }
+ return 0;
+}
+
+int
+rte_memseg_walk(rte_memseg_walk_t func, void *arg)
+{
+ int ret = 0;
+
+ /* do not allow allocations/frees/init while we iterate */
+ rte_mcfg_mem_read_lock();
+ ret = rte_memseg_walk_thread_unsafe(func, arg);
+ rte_mcfg_mem_read_unlock();
+
+ return ret;
+}
+
+int
+rte_memseg_list_walk_thread_unsafe(rte_memseg_list_walk_t func, void *arg)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ int i, ret = 0;
+
+ for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+ struct rte_memseg_list *msl = &mcfg->memsegs[i];
+
+ if (msl->base_va == NULL)
+ continue;
+
+ ret = func(msl, arg);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
+int
+rte_memseg_list_walk(rte_memseg_list_walk_t func, void *arg)
+{
+ int ret = 0;
+
+ /* do not allow allocations/frees/init while we iterate */
+ rte_mcfg_mem_read_lock();
+ ret = rte_memseg_list_walk_thread_unsafe(func, arg);
+ rte_mcfg_mem_read_unlock();
+
+ return ret;
+}
+
+int
+rte_memseg_get_fd_thread_unsafe(const struct rte_memseg *ms)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ struct rte_memseg_list *msl;
+ struct rte_fbarray *arr;
+ int msl_idx, seg_idx, ret;
+
+ if (ms == NULL) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+
+ msl = rte_mem_virt2memseg_list(ms->addr);
+ if (msl == NULL) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+ arr = &msl->memseg_arr;
+
+ msl_idx = msl - mcfg->memsegs;
+ seg_idx = rte_fbarray_find_idx(arr, ms);
+
+ if (!rte_fbarray_is_used(arr, seg_idx)) {
+ rte_errno = ENOENT;
+ return -1;
+ }
+
+ /* segment fd API is not supported for external segments */
+ if (msl->external) {
+ rte_errno = ENOTSUP;
+ return -1;
+ }
+
+ ret = eal_memalloc_get_seg_fd(msl_idx, seg_idx);
+ if (ret < 0) {
+ rte_errno = -ret;
+ ret = -1;
+ }
+ return ret;
+}
+
+int
+rte_memseg_get_fd(const struct rte_memseg *ms)
+{
+ int ret;
+
+ rte_mcfg_mem_read_lock();
+ ret = rte_memseg_get_fd_thread_unsafe(ms);
+ rte_mcfg_mem_read_unlock();
+
+ return ret;
+}
+
+int
+rte_memseg_get_fd_offset_thread_unsafe(const struct rte_memseg *ms,
+ size_t *offset)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ struct rte_memseg_list *msl;
+ struct rte_fbarray *arr;
+ int msl_idx, seg_idx, ret;
+
+ if (ms == NULL || offset == NULL) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+
+ msl = rte_mem_virt2memseg_list(ms->addr);
+ if (msl == NULL) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+ arr = &msl->memseg_arr;
+
+ msl_idx = msl - mcfg->memsegs;
+ seg_idx = rte_fbarray_find_idx(arr, ms);
+
+ if (!rte_fbarray_is_used(arr, seg_idx)) {
+ rte_errno = ENOENT;
+ return -1;
+ }
+
+ /* segment fd API is not supported for external segments */
+ if (msl->external) {
+ rte_errno = ENOTSUP;
+ return -1;
+ }
+
+ ret = eal_memalloc_get_seg_fd_offset(msl_idx, seg_idx, offset);
+ if (ret < 0) {
+ rte_errno = -ret;
+ ret = -1;
+ }
+ return ret;
+}
+
+int
+rte_memseg_get_fd_offset(const struct rte_memseg *ms, size_t *offset)
+{
+ int ret;
+
+ rte_mcfg_mem_read_lock();
+ ret = rte_memseg_get_fd_offset_thread_unsafe(ms, offset);
+ rte_mcfg_mem_read_unlock();
+
+ return ret;
+}
+
+int
+rte_extmem_register(void *va_addr, size_t len, rte_iova_t iova_addrs[],
+ unsigned int n_pages, size_t page_sz)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ unsigned int socket_id, n;
+ int ret = 0;
+
+ if (va_addr == NULL || page_sz == 0 || len == 0 ||
+ !rte_is_power_of_2(page_sz) ||
+ RTE_ALIGN(len, page_sz) != len ||
+ ((len / page_sz) != n_pages && iova_addrs != NULL) ||
+ !rte_is_aligned(va_addr, page_sz)) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+ rte_mcfg_mem_write_lock();
+
+ /* make sure the segment doesn't already exist */
+ if (malloc_heap_find_external_seg(va_addr, len) != NULL) {
+ rte_errno = EEXIST;
+ ret = -1;
+ goto unlock;
+ }
+
+ /* get next available socket ID */
+ socket_id = mcfg->next_socket_id;
+ if (socket_id > INT32_MAX) {
+ RTE_LOG(ERR, EAL, "Cannot assign new socket ID's\n");
+ rte_errno = ENOSPC;
+ ret = -1;
+ goto unlock;
+ }
+
+ /* we can create a new memseg */
+ n = len / page_sz;
+ if (malloc_heap_create_external_seg(va_addr, iova_addrs, n,
+ page_sz, "extmem", socket_id) == NULL) {
+ ret = -1;
+ goto unlock;
+ }
+
+ /* memseg list successfully created - increment next socket ID */
+ mcfg->next_socket_id++;
+unlock:
+ rte_mcfg_mem_write_unlock();
+ return ret;
+}
+
+int
+rte_extmem_unregister(void *va_addr, size_t len)
+{
+ struct rte_memseg_list *msl;
+ int ret = 0;
+
+ if (va_addr == NULL || len == 0) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+ rte_mcfg_mem_write_lock();
+
+ /* find our segment */
+ msl = malloc_heap_find_external_seg(va_addr, len);
+ if (msl == NULL) {
+ rte_errno = ENOENT;
+ ret = -1;
+ goto unlock;
+ }
+
+ ret = malloc_heap_destroy_external_seg(msl);
+unlock:
+ rte_mcfg_mem_write_unlock();
+ return ret;
+}
+
+static int
+sync_memory(void *va_addr, size_t len, bool attach)
+{
+ struct rte_memseg_list *msl;
+ int ret = 0;
+
+ if (va_addr == NULL || len == 0) {
+ rte_errno = EINVAL;
+ return -1;
+ }
+ rte_mcfg_mem_write_lock();
+
+ /* find our segment */
+ msl = malloc_heap_find_external_seg(va_addr, len);
+ if (msl == NULL) {
+ rte_errno = ENOENT;
+ ret = -1;
+ goto unlock;
+ }
+ if (attach)
+ ret = rte_fbarray_attach(&msl->memseg_arr);
+ else
+ ret = rte_fbarray_detach(&msl->memseg_arr);
+
+unlock:
+ rte_mcfg_mem_write_unlock();
+ return ret;
+}
+
+int
+rte_extmem_attach(void *va_addr, size_t len)
+{
+ return sync_memory(va_addr, len, true);
+}
+
+int
+rte_extmem_detach(void *va_addr, size_t len)
+{
+ return sync_memory(va_addr, len, false);
+}
+
+/* detach all EAL memory */
+int
+rte_eal_memory_detach(void)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ size_t page_sz = rte_mem_page_size();
+ unsigned int i;
+
+ rte_rwlock_write_lock(&mcfg->memory_hotplug_lock);
+
+ /* detach internal memory subsystem data first */
+ if (eal_memalloc_cleanup())
+ RTE_LOG(ERR, EAL, "Could not release memory subsystem data\n");
+
+ for (i = 0; i < RTE_DIM(mcfg->memsegs); i++) {
+ struct rte_memseg_list *msl = &mcfg->memsegs[i];
+
+ /* skip uninitialized segments */
+ if (msl->base_va == NULL)
+ continue;
+ /*
+ * external segments are supposed to be detached at this point,
+ * but if they aren't, we can't really do anything about it,
+ * because if we skip them here, they'll become invalid after
+ * we unmap the memconfig anyway. however, if this is externally
+ * referenced memory, we have no business unmapping it.
+ */
+ if (!msl->external)
+ if (rte_mem_unmap(msl->base_va, msl->len) != 0)
+ RTE_LOG(ERR, EAL, "Could not unmap memory: %s\n",
+ strerror(errno));
+
+ /*
+ * we are detaching the fbarray rather than destroying because
+ * other processes might still reference this fbarray, and we
+ * have no way of knowing if they still do.
+ */
+ if (rte_fbarray_detach(&msl->memseg_arr))
+ RTE_LOG(ERR, EAL, "Could not detach fbarray: %s\n",
+ rte_strerror(rte_errno));
+ }
+ rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock);
+
+ /*
+ * we've detached the memseg lists, so we can unmap the shared mem
+ * config - we can't zero it out because it might still be referenced
+ * by other processes.
+ */
+ rte_mem_unmap(mcfg, RTE_ALIGN(sizeof(*mcfg), page_sz));
+ rte_eal_get_configuration()->mem_config = NULL;
+
+ return 0;
+}
+
+/* init memory subsystem */
+int
+rte_eal_memory_init(void)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
+ int retval;
+ RTE_LOG(DEBUG, EAL, "Setting up physically contiguous memory...\n");
+
+ if (!mcfg)
+ return -1;
+
+ /* lock mem hotplug here, to prevent races while we init */
+ rte_mcfg_mem_read_lock();
+
+ if (rte_eal_memseg_init() < 0)
+ goto fail;
+
+ if (eal_memalloc_init() < 0)
+ goto fail;
+
+ retval = rte_eal_process_type() == RTE_PROC_PRIMARY ?
+ rte_eal_hugepage_init() :
+ rte_eal_hugepage_attach();
+ if (retval < 0)
+ goto fail;
+
+ if (internal_conf->no_shconf == 0 && rte_eal_memdevice_init() < 0)
+ goto fail;
+
+ return 0;
+fail:
+ rte_mcfg_mem_read_unlock();
+ return -1;
+}