eal: add option to limit memory allocation on sockets

Previously, it was possible to limit maximum amount of memory
allowed for allocation by creating validator callbacks. Although a
powerful tool, it's a bit of a hassle and requires modifying the
application for it to work with DPDK example applications.

Fix this by adding a new parameter "--socket-limit", with syntax
similar to "--socket-mem", which would set per-socket memory
allocation limits, and set up a default validator callback to deny
all allocations above the limit.

This option is incompatible with legacy mode, as validator callbacks
are not supported there.

Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>

diff --git a/doc/guides/linux_gsg/build_sample_apps.rst b/doc/guides/linux_gsg/build_sample_apps.rst
index 3623ddf..332424e 100644 (file)
--- a/doc/guides/linux_gsg/build_sample_apps.rst
+++ b/doc/guides/linux_gsg/build_sample_apps.rst
@@ -114,6 +114,10 @@ The EAL options are as follows:
   this memory will also be pinned (i.e. not released back to the system until
   application closes).
 
+* ``--socket-limit``:
+  Limit maximum memory available for allocation on each socket. Does not support
+  legacy memory mode.
+
 * ``-d``:
   Add a driver or driver directory to be loaded.
   The application should use this option to load the pmd drivers

diff --git a/doc/guides/prog_guide/env_abstraction_layer.rst b/doc/guides/prog_guide/env_abstraction_layer.rst
index a22640d..4c51efd 100644 (file)
--- a/doc/guides/prog_guide/env_abstraction_layer.rst
+++ b/doc/guides/prog_guide/env_abstraction_layer.rst
@@ -147,6 +147,10 @@ notified about memory allocations above specified threshold (and have a chance
 to deny them), allocation validator callbacks are also available via
 ``rte_mem_alloc_validator_callback_register()`` function.
 
+A default validator callback is provided by EAL, which can be enabled with a
+``--socket-limit`` command-line option, for a simple way to limit maximum amount
+of memory that can be used by DPDK application.
+
 .. note::
 
     In multiprocess scenario, all related processes (i.e. primary process, and

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index ecebb29..45ea01a 100644 (file)
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -70,6 +70,7 @@ eal_long_options[] = {
        {OPT_PCI_WHITELIST,     1, NULL, OPT_PCI_WHITELIST_NUM    },
        {OPT_PROC_TYPE,         1, NULL, OPT_PROC_TYPE_NUM        },
        {OPT_SOCKET_MEM,        1, NULL, OPT_SOCKET_MEM_NUM       },
+       {OPT_SOCKET_LIMIT,      1, NULL, OPT_SOCKET_LIMIT_NUM     },
        {OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
        {OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
        {OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
@@ -179,6 +180,10 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
        /* zero out the NUMA config */
        for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
                internal_cfg->socket_mem[i] = 0;
+       internal_cfg->force_socket_limits = 0;
+       /* zero out the NUMA limits config */
+       for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
+               internal_cfg->socket_limit[i] = 0;
        /* zero out hugedir descriptors */
        for (i = 0; i < MAX_HUGEPAGE_SIZES; i++) {
                memset(&internal_cfg->hugepage_info[i], 0,
@@ -1322,6 +1327,11 @@ eal_check_common_options(struct internal_config *internal_cfg)
                        "be specified together with --"OPT_NO_HUGE"\n");
                return -1;
        }
+       if (internal_config.force_socket_limits && internal_config.legacy_mem) {
+               RTE_LOG(ERR, EAL, "Option --"OPT_SOCKET_LIMIT
+                       " is only supported in non-legacy memory mode\n");
+               return -1;
+       }
 
        return 0;
 }

diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index c4cbf3a..d66cd03 100644 (file)
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -46,6 +46,8 @@ struct internal_config {
        /** true to try allocating memory on specific sockets */
        volatile unsigned force_sockets;
        volatile uint64_t socket_mem[RTE_MAX_NUMA_NODES]; /**< amount of memory per socket */
+       volatile unsigned force_socket_limits;
+       volatile uint64_t socket_limit[RTE_MAX_NUMA_NODES]; /**< limit amount of memory per socket */
        uintptr_t base_virtaddr;          /**< base address to try and reserve memory from */
        volatile unsigned legacy_mem;
        /**< true to enable legacy memory behavior (no dynamic allocation,

diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 211ae06..6d92f64 100644 (file)
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -47,6 +47,8 @@ enum {
        OPT_NO_SHCONF_NUM,
 #define OPT_SOCKET_MEM        "socket-mem"
        OPT_SOCKET_MEM_NUM,
+#define OPT_SOCKET_LIMIT        "socket-limit"
+       OPT_SOCKET_LIMIT_NUM,
 #define OPT_SYSLOG            "syslog"
        OPT_SYSLOG_NUM,
 #define OPT_VDEV              "vdev"

diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 987b57f..ec7cea5 100644 (file)
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -405,6 +405,7 @@ eal_usage(const char *prgname)
        eal_common_usage();
        printf("EAL Linux options:\n"
               "  --"OPT_SOCKET_MEM"        Memory to allocate on sockets (comma separated values)\n"
+              "  --"OPT_SOCKET_LIMIT"      Limit memory allocation on sockets (comma separated values)\n"
               "  --"OPT_HUGE_DIR"          Directory where hugetlbfs is mounted\n"
               "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
               "  --"OPT_BASE_VIRTADDR"     Base virtual address\n"
@@ -434,46 +435,45 @@ rte_set_application_usage_hook( rte_usage_hook_t usage_func )
 }
 
 static int
-eal_parse_socket_mem(char *socket_mem)
+eal_parse_socket_arg(char *strval, volatile uint64_t *socket_arg)
 {
        char * arg[RTE_MAX_NUMA_NODES];
        char *end;
        int arg_num, i, len;
        uint64_t total_mem = 0;
 
-       len = strnlen(socket_mem, SOCKET_MEM_STRLEN);
+       len = strnlen(strval, SOCKET_MEM_STRLEN);
        if (len == SOCKET_MEM_STRLEN) {
                RTE_LOG(ERR, EAL, "--socket-mem is too long\n");
                return -1;
        }
 
        /* all other error cases will be caught later */
-       if (!isdigit(socket_mem[len-1]))
+       if (!isdigit(strval[len-1]))
                return -1;
 
        /* split the optarg into separate socket values */
-       arg_num = rte_strsplit(socket_mem, len,
+       arg_num = rte_strsplit(strval, len,
                        arg, RTE_MAX_NUMA_NODES, ',');
 
        /* if split failed, or 0 arguments */
        if (arg_num <= 0)
                return -1;
 
-       internal_config.force_sockets = 1;
-
        /* parse each defined socket option */
        errno = 0;
        for (i = 0; i < arg_num; i++) {
+               uint64_t val;
                end = NULL;
-               internal_config.socket_mem[i] = strtoull(arg[i], &end, 10);
+               val = strtoull(arg[i], &end, 10);
 
                /* check for invalid input */
                if ((errno != 0)  ||
                                (arg[i][0] == '\0') || (end == NULL) || (*end != '\0'))
                        return -1;
-               internal_config.socket_mem[i] *= 1024ULL;
-               internal_config.socket_mem[i] *= 1024ULL;
-               total_mem += internal_config.socket_mem[i];
+               val <<= 20;
+               total_mem += val;
+               socket_arg[i] = val;
        }
 
        /* check if we have a positive amount of total memory */
@@ -621,13 +621,27 @@ eal_parse_args(int argc, char **argv)
                        break;
 
                case OPT_SOCKET_MEM_NUM:
-                       if (eal_parse_socket_mem(optarg) < 0) {
+                       if (eal_parse_socket_arg(optarg,
+                                       internal_config.socket_mem) < 0) {
                                RTE_LOG(ERR, EAL, "invalid parameters for --"
                                                OPT_SOCKET_MEM "\n");
                                eal_usage(prgname);
                                ret = -1;
                                goto out;
                        }
+                       internal_config.force_sockets = 1;
+                       break;
+
+               case OPT_SOCKET_LIMIT_NUM:
+                       if (eal_parse_socket_arg(optarg,
+                                       internal_config.socket_limit) < 0) {
+                               RTE_LOG(ERR, EAL, "invalid parameters for --"
+                                               OPT_SOCKET_LIMIT "\n");
+                               eal_usage(prgname);
+                               ret = -1;
+                               goto out;
+                       }
+                       internal_config.force_socket_limits = 1;
                        break;
 
                case OPT_BASE_VIRTADDR_NUM:

diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c
index b8c8a59..a48a350 100644 (file)
--- a/lib/librte_eal/linuxapp/eal/eal_memory.c
+++ b/lib/librte_eal/linuxapp/eal/eal_memory.c
@@ -1632,6 +1632,15 @@ hugepage_count_walk(const struct rte_memseg_list *msl, void *arg)
        return 0;
 }
 
+static int
+limits_callback(int socket_id, size_t cur_limit, size_t new_len)
+{
+       RTE_SET_USED(socket_id);
+       RTE_SET_USED(cur_limit);
+       RTE_SET_USED(new_len);
+       return -1;
+}
+
 static int
 eal_hugepage_init(void)
 {
@@ -1715,6 +1724,18 @@ eal_hugepage_init(void)
                        free(pages);
                }
        }
+       /* if socket limits were specified, set them */
+       if (internal_config.force_socket_limits) {
+               unsigned int i;
+               for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
+                       uint64_t limit = internal_config.socket_limit[i];
+                       if (limit == 0)
+                               continue;
+                       if (rte_mem_alloc_validator_register("socket-limit",
+                                       limits_callback, i, limit))
+                               RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n");
+               }
+       }
        return 0;
 }