From 6258f1c942c3e0b7bb715158cc266eede6521ddf Mon Sep 17 00:00:00 2001 From: Anatoly Burakov Date: Fri, 20 Jun 2014 16:42:16 +0100 Subject: [PATCH] eal: map shared config into exact same address as primary process Shared config is shared across primary and secondary processes. However,when using rte_malloc, the malloc elements keep references to the heap inside themselves. This heap reference might not be referencing a local heap because the heap reference points to the heap of whatever process has allocated that malloc element. Therefore, there can be situations when malloc elements in a given heap actually reference different addresses for the same heap - depending on which process has allocated the element. This can lead to segmentation faults when dealing with malloc elements allocated on the same heap by different processes. To fix this problem, heaps will now have the same addresses across processes. In order to achieve that, a new field in a shared mem_config (a structure that holds the heaps, and which is shared across processes) was added to keep the address of where this config is mapped in the primary process. Secondary process will now map the config in two stages - first, it'll map it into an arbitrary address and read the address the primary process has allocated for the shared config. Then, the config is unmapped and re-mapped using the address previously read. Signed-off-by: Anatoly Burakov Acked-by: Konstantin Ananyev --- .../common/include/rte_eal_memconfig.h | 5 +++ lib/librte_eal/linuxapp/eal/eal.c | 44 ++++++++++++++++--- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/lib/librte_eal/common/include/rte_eal_memconfig.h b/lib/librte_eal/common/include/rte_eal_memconfig.h index 30ce6fc090..d6359e51c6 100644 --- a/lib/librte_eal/common/include/rte_eal_memconfig.h +++ b/lib/librte_eal/common/include/rte_eal_memconfig.h @@ -89,6 +89,11 @@ struct rte_mem_config { /* Heaps of Malloc per socket */ struct malloc_heap malloc_heaps[RTE_MAX_NUMA_NODES]; + + /* address of mem_config in primary process. used to map shared config into + * exact same address the primary process maps it. + */ + uint64_t mem_cfg_addr; } __attribute__((__packed__)); diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c index 573fd06477..5d2adb9b31 100644 --- a/lib/librte_eal/linuxapp/eal/eal.c +++ b/lib/librte_eal/linuxapp/eal/eal.c @@ -240,13 +240,19 @@ rte_eal_config_create(void) } memcpy(rte_mem_cfg_addr, &early_mem_config, sizeof(early_mem_config)); rte_config.mem_config = (struct rte_mem_config *) rte_mem_cfg_addr; + + /* store address of the config in the config itself so that secondary + * processes could later map the config into this exact location */ + rte_config.mem_config->mem_cfg_addr = (uintptr_t) rte_mem_cfg_addr; + } /* attach to an existing shared memory config */ static void rte_eal_config_attach(void) { - void *rte_mem_cfg_addr; + struct rte_mem_config *mem_config; + const char *pathname = eal_runtime_config_path(); if (internal_config.no_shconf) @@ -258,13 +264,40 @@ rte_eal_config_attach(void) rte_panic("Cannot open '%s' for rte_mem_config\n", pathname); } - rte_mem_cfg_addr = mmap(NULL, sizeof(*rte_config.mem_config), - PROT_READ | PROT_WRITE, MAP_SHARED, mem_cfg_fd, 0); + /* map it as read-only first */ + mem_config = (struct rte_mem_config *) mmap(NULL, sizeof(*mem_config), + PROT_READ, MAP_SHARED, mem_cfg_fd, 0); + if (mem_config == MAP_FAILED) + rte_panic("Cannot mmap memory for rte_config\n"); + + rte_config.mem_config = mem_config; +} + +/* reattach the shared config at exact memory location primary process has it */ +static void +rte_eal_config_reattach(void) +{ + struct rte_mem_config *mem_config; + void *rte_mem_cfg_addr; + + if (internal_config.no_shconf) + return; + + /* save the address primary process has mapped shared config to */ + rte_mem_cfg_addr = (void *) (uintptr_t) rte_config.mem_config->mem_cfg_addr; + + /* unmap original config */ + munmap(rte_config.mem_config, sizeof(struct rte_mem_config)); + + /* remap the config at proper address */ + mem_config = (struct rte_mem_config *) mmap(rte_mem_cfg_addr, + sizeof(*mem_config), PROT_READ | PROT_WRITE, MAP_SHARED, + mem_cfg_fd, 0); close(mem_cfg_fd); - if (rte_mem_cfg_addr == MAP_FAILED) + if (mem_config == MAP_FAILED || mem_config != rte_mem_cfg_addr) rte_panic("Cannot mmap memory for rte_config\n"); - rte_config.mem_config = (struct rte_mem_config *) rte_mem_cfg_addr; + rte_config.mem_config = mem_config; } /* Detect if we are a primary or a secondary process */ @@ -302,6 +335,7 @@ rte_config_init(void) case RTE_PROC_SECONDARY: rte_eal_config_attach(); rte_eal_mcfg_wait_complete(rte_config.mem_config); + rte_eal_config_reattach(); break; case RTE_PROC_AUTO: case RTE_PROC_INVALID: -- 2.20.1