From e4348122a42a15fd03d1221a1c34615d5b68add6 Mon Sep 17 00:00:00 2001 From: Anatoly Burakov Date: Thu, 31 May 2018 18:35:33 +0100 Subject: [PATCH] eal: add option to limit memory allocation on sockets Previously, it was possible to limit maximum amount of memory allowed for allocation by creating validator callbacks. Although a powerful tool, it's a bit of a hassle and requires modifying the application for it to work with DPDK example applications. Fix this by adding a new parameter "--socket-limit", with syntax similar to "--socket-mem", which would set per-socket memory allocation limits, and set up a default validator callback to deny all allocations above the limit. This option is incompatible with legacy mode, as validator callbacks are not supported there. Signed-off-by: Anatoly Burakov --- doc/guides/linux_gsg/build_sample_apps.rst | 4 +++ .../prog_guide/env_abstraction_layer.rst | 4 +++ lib/librte_eal/common/eal_common_options.c | 10 ++++++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/linuxapp/eal/eal.c | 36 +++++++++++++------ lib/librte_eal/linuxapp/eal/eal_memory.c | 21 +++++++++++ 7 files changed, 68 insertions(+), 11 deletions(-) diff --git a/doc/guides/linux_gsg/build_sample_apps.rst b/doc/guides/linux_gsg/build_sample_apps.rst index 3623ddf46e..332424e05c 100644 --- a/doc/guides/linux_gsg/build_sample_apps.rst +++ b/doc/guides/linux_gsg/build_sample_apps.rst @@ -114,6 +114,10 @@ The EAL options are as follows: this memory will also be pinned (i.e. not released back to the system until application closes). +* ``--socket-limit``: + Limit maximum memory available for allocation on each socket. Does not support + legacy memory mode. + * ``-d``: Add a driver or driver directory to be loaded. The application should use this option to load the pmd drivers diff --git a/doc/guides/prog_guide/env_abstraction_layer.rst b/doc/guides/prog_guide/env_abstraction_layer.rst index a22640d29f..4c51efd42c 100644 --- a/doc/guides/prog_guide/env_abstraction_layer.rst +++ b/doc/guides/prog_guide/env_abstraction_layer.rst @@ -147,6 +147,10 @@ notified about memory allocations above specified threshold (and have a chance to deny them), allocation validator callbacks are also available via ``rte_mem_alloc_validator_callback_register()`` function. +A default validator callback is provided by EAL, which can be enabled with a +``--socket-limit`` command-line option, for a simple way to limit maximum amount +of memory that can be used by DPDK application. + .. note:: In multiprocess scenario, all related processes (i.e. primary process, and diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c index ecebb29231..45ea01a8bc 100644 --- a/lib/librte_eal/common/eal_common_options.c +++ b/lib/librte_eal/common/eal_common_options.c @@ -70,6 +70,7 @@ eal_long_options[] = { {OPT_PCI_WHITELIST, 1, NULL, OPT_PCI_WHITELIST_NUM }, {OPT_PROC_TYPE, 1, NULL, OPT_PROC_TYPE_NUM }, {OPT_SOCKET_MEM, 1, NULL, OPT_SOCKET_MEM_NUM }, + {OPT_SOCKET_LIMIT, 1, NULL, OPT_SOCKET_LIMIT_NUM }, {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, @@ -179,6 +180,10 @@ eal_reset_internal_config(struct internal_config *internal_cfg) /* zero out the NUMA config */ for (i = 0; i < RTE_MAX_NUMA_NODES; i++) internal_cfg->socket_mem[i] = 0; + internal_cfg->force_socket_limits = 0; + /* zero out the NUMA limits config */ + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) + internal_cfg->socket_limit[i] = 0; /* zero out hugedir descriptors */ for (i = 0; i < MAX_HUGEPAGE_SIZES; i++) { memset(&internal_cfg->hugepage_info[i], 0, @@ -1322,6 +1327,11 @@ eal_check_common_options(struct internal_config *internal_cfg) "be specified together with --"OPT_NO_HUGE"\n"); return -1; } + if (internal_config.force_socket_limits && internal_config.legacy_mem) { + RTE_LOG(ERR, EAL, "Option --"OPT_SOCKET_LIMIT + " is only supported in non-legacy memory mode\n"); + return -1; + } return 0; } diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h index c4cbf3acda..d66cd03134 100644 --- a/lib/librte_eal/common/eal_internal_cfg.h +++ b/lib/librte_eal/common/eal_internal_cfg.h @@ -46,6 +46,8 @@ struct internal_config { /** true to try allocating memory on specific sockets */ volatile unsigned force_sockets; volatile uint64_t socket_mem[RTE_MAX_NUMA_NODES]; /**< amount of memory per socket */ + volatile unsigned force_socket_limits; + volatile uint64_t socket_limit[RTE_MAX_NUMA_NODES]; /**< limit amount of memory per socket */ uintptr_t base_virtaddr; /**< base address to try and reserve memory from */ volatile unsigned legacy_mem; /**< true to enable legacy memory behavior (no dynamic allocation, diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h index 211ae06aef..6d92f64a8f 100644 --- a/lib/librte_eal/common/eal_options.h +++ b/lib/librte_eal/common/eal_options.h @@ -47,6 +47,8 @@ enum { OPT_NO_SHCONF_NUM, #define OPT_SOCKET_MEM "socket-mem" OPT_SOCKET_MEM_NUM, +#define OPT_SOCKET_LIMIT "socket-limit" + OPT_SOCKET_LIMIT_NUM, #define OPT_SYSLOG "syslog" OPT_SYSLOG_NUM, #define OPT_VDEV "vdev" diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c index 987b57f877..ec7cea55d3 100644 --- a/lib/librte_eal/linuxapp/eal/eal.c +++ b/lib/librte_eal/linuxapp/eal/eal.c @@ -405,6 +405,7 @@ eal_usage(const char *prgname) eal_common_usage(); printf("EAL Linux options:\n" " --"OPT_SOCKET_MEM" Memory to allocate on sockets (comma separated values)\n" + " --"OPT_SOCKET_LIMIT" Limit memory allocation on sockets (comma separated values)\n" " --"OPT_HUGE_DIR" Directory where hugetlbfs is mounted\n" " --"OPT_FILE_PREFIX" Prefix for hugepage filenames\n" " --"OPT_BASE_VIRTADDR" Base virtual address\n" @@ -434,46 +435,45 @@ rte_set_application_usage_hook( rte_usage_hook_t usage_func ) } static int -eal_parse_socket_mem(char *socket_mem) +eal_parse_socket_arg(char *strval, volatile uint64_t *socket_arg) { char * arg[RTE_MAX_NUMA_NODES]; char *end; int arg_num, i, len; uint64_t total_mem = 0; - len = strnlen(socket_mem, SOCKET_MEM_STRLEN); + len = strnlen(strval, SOCKET_MEM_STRLEN); if (len == SOCKET_MEM_STRLEN) { RTE_LOG(ERR, EAL, "--socket-mem is too long\n"); return -1; } /* all other error cases will be caught later */ - if (!isdigit(socket_mem[len-1])) + if (!isdigit(strval[len-1])) return -1; /* split the optarg into separate socket values */ - arg_num = rte_strsplit(socket_mem, len, + arg_num = rte_strsplit(strval, len, arg, RTE_MAX_NUMA_NODES, ','); /* if split failed, or 0 arguments */ if (arg_num <= 0) return -1; - internal_config.force_sockets = 1; - /* parse each defined socket option */ errno = 0; for (i = 0; i < arg_num; i++) { + uint64_t val; end = NULL; - internal_config.socket_mem[i] = strtoull(arg[i], &end, 10); + val = strtoull(arg[i], &end, 10); /* check for invalid input */ if ((errno != 0) || (arg[i][0] == '\0') || (end == NULL) || (*end != '\0')) return -1; - internal_config.socket_mem[i] *= 1024ULL; - internal_config.socket_mem[i] *= 1024ULL; - total_mem += internal_config.socket_mem[i]; + val <<= 20; + total_mem += val; + socket_arg[i] = val; } /* check if we have a positive amount of total memory */ @@ -621,13 +621,27 @@ eal_parse_args(int argc, char **argv) break; case OPT_SOCKET_MEM_NUM: - if (eal_parse_socket_mem(optarg) < 0) { + if (eal_parse_socket_arg(optarg, + internal_config.socket_mem) < 0) { RTE_LOG(ERR, EAL, "invalid parameters for --" OPT_SOCKET_MEM "\n"); eal_usage(prgname); ret = -1; goto out; } + internal_config.force_sockets = 1; + break; + + case OPT_SOCKET_LIMIT_NUM: + if (eal_parse_socket_arg(optarg, + internal_config.socket_limit) < 0) { + RTE_LOG(ERR, EAL, "invalid parameters for --" + OPT_SOCKET_LIMIT "\n"); + eal_usage(prgname); + ret = -1; + goto out; + } + internal_config.force_socket_limits = 1; break; case OPT_BASE_VIRTADDR_NUM: diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c index b8c8a59e07..a48a3502c2 100644 --- a/lib/librte_eal/linuxapp/eal/eal_memory.c +++ b/lib/librte_eal/linuxapp/eal/eal_memory.c @@ -1632,6 +1632,15 @@ hugepage_count_walk(const struct rte_memseg_list *msl, void *arg) return 0; } +static int +limits_callback(int socket_id, size_t cur_limit, size_t new_len) +{ + RTE_SET_USED(socket_id); + RTE_SET_USED(cur_limit); + RTE_SET_USED(new_len); + return -1; +} + static int eal_hugepage_init(void) { @@ -1715,6 +1724,18 @@ eal_hugepage_init(void) free(pages); } } + /* if socket limits were specified, set them */ + if (internal_config.force_socket_limits) { + unsigned int i; + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) { + uint64_t limit = internal_config.socket_limit[i]; + if (limit == 0) + continue; + if (rte_mem_alloc_validator_register("socket-limit", + limits_callback, i, limit)) + RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n"); + } + } return 0; } -- 2.20.1