#include "eal_private.h"
#include "eal_internal_cfg.h"
#include "eal_memcfg.h"
+#include "eal_options.h"
#include "malloc_heap.h"
/*
uint64_t map_sz;
void *mapped_addr, *aligned_addr;
uint8_t try = 0;
+ struct internal_config *internal_conf =
+ eal_get_internal_configuration();
if (system_page_sz == 0)
system_page_sz = rte_mem_page_size();
allow_shrink = (flags & EAL_VIRTUAL_AREA_ALLOW_SHRINK) > 0;
unmap = (flags & EAL_VIRTUAL_AREA_UNMAP) > 0;
- if (next_baseaddr == NULL && internal_config.base_virtaddr != 0 &&
+ if (next_baseaddr == NULL && internal_conf->base_virtaddr != 0 &&
rte_eal_process_type() == RTE_PROC_PRIMARY)
- next_baseaddr = (void *) internal_config.base_virtaddr;
+ next_baseaddr = (void *) internal_conf->base_virtaddr;
#ifdef RTE_ARCH_64
- if (next_baseaddr == NULL && internal_config.base_virtaddr == 0 &&
+ if (next_baseaddr == NULL && internal_conf->base_virtaddr == 0 &&
rte_eal_process_type() == RTE_PROC_PRIMARY)
next_baseaddr = (void *) eal_get_baseaddr();
#endif
return aligned_addr;
}
+int
+eal_memseg_list_init_named(struct rte_memseg_list *msl, const char *name,
+ uint64_t page_sz, int n_segs, int socket_id, bool heap)
+{
+ if (rte_fbarray_init(&msl->memseg_arr, name, n_segs,
+ sizeof(struct rte_memseg))) {
+ RTE_LOG(ERR, EAL, "Cannot allocate memseg list: %s\n",
+ rte_strerror(rte_errno));
+ return -1;
+ }
+
+ msl->page_sz = page_sz;
+ msl->socket_id = socket_id;
+ msl->base_va = NULL;
+ msl->heap = heap;
+
+ RTE_LOG(DEBUG, EAL,
+ "Memseg list allocated at socket %i, page size 0x%"PRIx64"kB\n",
+ socket_id, page_sz >> 10);
+
+ return 0;
+}
+
+int
+eal_memseg_list_init(struct rte_memseg_list *msl, uint64_t page_sz,
+ int n_segs, int socket_id, int type_msl_idx, bool heap)
+{
+ char name[RTE_FBARRAY_NAME_LEN];
+
+ snprintf(name, sizeof(name), MEMSEG_LIST_FMT, page_sz >> 10, socket_id,
+ type_msl_idx);
+
+ return eal_memseg_list_init_named(
+ msl, name, page_sz, n_segs, socket_id, heap);
+}
+
+int
+eal_memseg_list_alloc(struct rte_memseg_list *msl, int reserve_flags)
+{
+ size_t page_sz, mem_sz;
+ void *addr;
+
+ page_sz = msl->page_sz;
+ mem_sz = page_sz * msl->memseg_arr.len;
+
+ addr = eal_get_virtual_area(
+ msl->base_va, &mem_sz, page_sz, 0, reserve_flags);
+ if (addr == NULL) {
+#ifndef RTE_EXEC_ENV_WINDOWS
+ /* The hint would be misleading on Windows, because address
+ * is by default system-selected (base VA = 0).
+ * However, this function is called from many places,
+ * including common code, so don't duplicate the message.
+ */
+ if (rte_errno == EADDRNOTAVAIL)
+ RTE_LOG(ERR, EAL, "Cannot reserve %llu bytes at [%p] - "
+ "please use '--" OPT_BASE_VIRTADDR "' option\n",
+ (unsigned long long)mem_sz, msl->base_va);
+#endif
+ return -1;
+ }
+ msl->base_va = addr;
+ msl->len = mem_sz;
+
+ RTE_LOG(DEBUG, EAL, "VA reserved for memseg list at %p, size %zx\n",
+ addr, mem_sz);
+
+ return 0;
+}
+
+void
+eal_memseg_list_populate(struct rte_memseg_list *msl, void *addr, int n_segs)
+{
+ size_t page_sz = msl->page_sz;
+ int i;
+
+ for (i = 0; i < n_segs; i++) {
+ struct rte_fbarray *arr = &msl->memseg_arr;
+ struct rte_memseg *ms = rte_fbarray_get(arr, i);
+
+ if (rte_eal_iova_mode() == RTE_IOVA_VA)
+ ms->iova = (uintptr_t)addr;
+ else
+ ms->iova = RTE_BAD_IOVA;
+ ms->addr = addr;
+ ms->hugepage_sz = page_sz;
+ ms->socket_id = 0;
+ ms->len = page_sz;
+
+ rte_fbarray_set_used(arr, i);
+
+ addr = RTE_PTR_ADD(addr, page_sz);
+ }
+}
+
static struct rte_memseg *
virt2memseg(const void *addr, const struct rte_memseg_list *msl)
{
rte_mem_iova2virt(rte_iova_t iova)
{
struct virtiova vi;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
memset(&vi, 0, sizeof(vi));
/* for legacy mem, we can get away with scanning VA-contiguous segments,
* as we know they are PA-contiguous as well
*/
- if (internal_config.legacy_mem)
+ if (internal_conf->legacy_mem)
rte_memseg_contig_walk(find_virt_legacy, &vi);
else
rte_memseg_walk(find_virt, &vi);
rte_mem_event_callback_register(const char *name, rte_mem_event_callback_t clb,
void *arg)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
/* FreeBSD boots with legacy mem enabled by default */
- if (internal_config.legacy_mem) {
+ if (internal_conf->legacy_mem) {
RTE_LOG(DEBUG, EAL, "Registering mem event callbacks not supported\n");
rte_errno = ENOTSUP;
return -1;
int
rte_mem_event_callback_unregister(const char *name, void *arg)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
/* FreeBSD boots with legacy mem enabled by default */
- if (internal_config.legacy_mem) {
+ if (internal_conf->legacy_mem) {
RTE_LOG(DEBUG, EAL, "Registering mem event callbacks not supported\n");
rte_errno = ENOTSUP;
return -1;
rte_mem_alloc_validator_register(const char *name,
rte_mem_alloc_validator_t clb, int socket_id, size_t limit)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
/* FreeBSD boots with legacy mem enabled by default */
- if (internal_config.legacy_mem) {
+ if (internal_conf->legacy_mem) {
RTE_LOG(DEBUG, EAL, "Registering mem alloc validators not supported\n");
rte_errno = ENOTSUP;
return -1;
int
rte_mem_alloc_validator_unregister(const char *name, int socket_id)
{
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
/* FreeBSD boots with legacy mem enabled by default */
- if (internal_config.legacy_mem) {
+ if (internal_conf->legacy_mem) {
RTE_LOG(DEBUG, EAL, "Registering mem alloc validators not supported\n");
rte_errno = ENOTSUP;
return -1;
rte_eal_memdevice_init(void)
{
struct rte_config *config;
+ const struct internal_config *internal_conf;
if (rte_eal_process_type() == RTE_PROC_SECONDARY)
return 0;
+ internal_conf = eal_get_internal_configuration();
config = rte_eal_get_configuration();
- config->mem_config->nchannel = internal_config.force_nchannel;
- config->mem_config->nrank = internal_config.force_nrank;
+ config->mem_config->nchannel = internal_conf->force_nchannel;
+ config->mem_config->nrank = internal_conf->force_nrank;
return 0;
}
return sync_memory(va_addr, len, false);
}
+/* detach all EAL memory */
+int
+rte_eal_memory_detach(void)
+{
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ size_t page_sz = rte_mem_page_size();
+ unsigned int i;
+
+ rte_rwlock_write_lock(&mcfg->memory_hotplug_lock);
+
+ /* detach internal memory subsystem data first */
+ if (eal_memalloc_cleanup())
+ RTE_LOG(ERR, EAL, "Could not release memory subsystem data\n");
+
+ for (i = 0; i < RTE_DIM(mcfg->memsegs); i++) {
+ struct rte_memseg_list *msl = &mcfg->memsegs[i];
+
+ /* skip uninitialized segments */
+ if (msl->base_va == NULL)
+ continue;
+ /*
+ * external segments are supposed to be detached at this point,
+ * but if they aren't, we can't really do anything about it,
+ * because if we skip them here, they'll become invalid after
+ * we unmap the memconfig anyway. however, if this is externally
+ * referenced memory, we have no business unmapping it.
+ */
+ if (!msl->external)
+ if (rte_mem_unmap(msl->base_va, msl->len) != 0)
+ RTE_LOG(ERR, EAL, "Could not unmap memory: %s\n",
+ strerror(errno));
+
+ /*
+ * we are detaching the fbarray rather than destroying because
+ * other processes might still reference this fbarray, and we
+ * have no way of knowing if they still do.
+ */
+ if (rte_fbarray_detach(&msl->memseg_arr))
+ RTE_LOG(ERR, EAL, "Could not detach fbarray: %s\n",
+ rte_strerror(rte_errno));
+ }
+ rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock);
+
+ /*
+ * we've detached the memseg lists, so we can unmap the shared mem
+ * config - we can't zero it out because it might still be referenced
+ * by other processes.
+ */
+ rte_mem_unmap(mcfg, RTE_ALIGN(sizeof(*mcfg), page_sz));
+ rte_eal_get_configuration()->mem_config = NULL;
+
+ return 0;
+}
+
/* init memory subsystem */
int
rte_eal_memory_init(void)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct internal_config *internal_conf =
+ eal_get_internal_configuration();
+
int retval;
RTE_LOG(DEBUG, EAL, "Setting up physically contiguous memory...\n");
if (retval < 0)
goto fail;
- if (internal_config.no_shconf == 0 && rte_eal_memdevice_init() < 0)
+ if (internal_conf->no_shconf == 0 && rte_eal_memdevice_init() < 0)
goto fail;
return 0;