From 85d9252e55f2a0cec836c2334ecc32ecf999c52a Mon Sep 17 00:00:00 2001 From: Michael Baum Date: Tue, 28 Jun 2022 17:58:40 +0300 Subject: [PATCH] net/mlx5: add test for remote PD and CTX Add mlx5 internal option in testpmd similar to run-time function "port attach" which adds another parameter named "socket" for attaching port and add 2 devargs before. The arguments are "cmd_fd" and "pd_handle" using to import device created out of PMD. Testpmd application import it using IPC, and updates the devargs list before attaching. These arguments were added in the commit 9d936f4f1a5e ("common/mlx5: support remote PD and CTX") The syntax is: testpmd> mlx5 port attach (identifier) socket=(path) Where "path" is the IPC socket path agreed on the remote process. Signed-off-by: Michael Baum Reviewed-by: Thomas Monjalon Acked-by: Matan Azrad --- doc/guides/nics/mlx5.rst | 47 +++++ doc/guides/testpmd_app_ug/testpmd_funcs.rst | 2 + drivers/net/mlx5/mlx5_testpmd.c | 220 ++++++++++++++++++++ 3 files changed, 269 insertions(+) diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst index bc2bd2c8a6..cd3a613640 100644 --- a/doc/guides/nics/mlx5.rst +++ b/doc/guides/nics/mlx5.rst @@ -1793,3 +1793,50 @@ and disables ``avail_thresh_triggered``. .. code-block:: console testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 50 + + +Testpmd +------- + +port attach with socket path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is possible to allocate a port with ``libibverbs`` from external application. +For importing the external port with extra device arguments, +there is a specific testpmd command +similar to :ref:`port attach command `:: + + testpmd> mlx5 port attach (identifier) socket=(path) + +where: + +* ``identifier``: device identifier with optional parameters + as same as :ref:`port attach command `. +* ``path``: path to IPC server socket created by the external application. + +This command performs: + +#. Open IPC client socket using the given path, and connect it. + +#. Import ibverbs context and ibverbs protection domain. + +#. Add two device arguments for context (``cmd_fd``) + and protection domain (``pd_handle``) to the device identifier. + See :ref:`mlx5 driver options ` for more + information about these device arguments. + +#. Call the regular ``port attach`` function with updated identifier. + +For example, to attach a port whose PCI address is ``0000:0a:00.0`` +and its socket path is ``/var/run/import_ipc_socket``: + +.. code-block:: console + + testpmd> mlx5 port attach 0000:0a:00.0 socket=/var/run/import_ipc_socket + testpmd: MLX5 socket path is /var/run/import_ipc_socket + testpmd: Attach port with extra devargs 0000:0a:00.0,cmd_fd=40,pd_handle=1 + Attaching a new port... + EAL: Probe PCI driver: mlx5_pci (15b3:101d) device: 0000:0a:00.0 (socket 0) + Port 0 is attached. Now total ports is 1 + Done + diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst index b0bb912455..330e34427d 100644 --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst @@ -1882,6 +1882,8 @@ The following sections show functions for configuring ports. Port configuration changes only become active when forwarding is started/restarted. +.. _port_attach: + port attach ~~~~~~~~~~~ diff --git a/drivers/net/mlx5/mlx5_testpmd.c b/drivers/net/mlx5/mlx5_testpmd.c index 4f9826496d..463ee8e764 100644 --- a/drivers/net/mlx5/mlx5_testpmd.c +++ b/drivers/net/mlx5/mlx5_testpmd.c @@ -6,6 +6,11 @@ #include #include #include +#include +#ifndef RTE_EXEC_ENV_WINDOWS +#include +#include +#endif #include #include @@ -14,6 +19,7 @@ #include #include #include + #include "mlx5_testpmd.h" #include "testpmd.h" @@ -111,6 +117,162 @@ mlx5_test_set_port_host_shaper(uint16_t port_id, uint16_t avail_thresh_triggered return 0; } +#ifndef RTE_EXEC_ENV_WINDOWS +static const char* +mlx5_test_get_socket_path(char *extend) +{ + if (strstr(extend, "socket=") == extend) { + const char *socket_path = strchr(extend, '=') + 1; + + TESTPMD_LOG(DEBUG, "MLX5 socket path is %s\n", socket_path); + return socket_path; + } + + TESTPMD_LOG(ERR, "Failed to extract a valid socket path from %s\n", + extend); + return NULL; +} + +static int +mlx5_test_extend_devargs(char *identifier, char *extend) +{ + struct sockaddr_un un = { + .sun_family = AF_UNIX, + }; + int cmd_fd; + int pd_handle; + struct iovec iov = { + .iov_base = &pd_handle, + .iov_len = sizeof(int), + }; + union { + char buf[CMSG_SPACE(sizeof(int))]; + struct cmsghdr align; + } control; + struct msghdr msgh = { + .msg_iov = NULL, + .msg_iovlen = 0, + }; + struct cmsghdr *cmsg; + const char *path = mlx5_test_get_socket_path(extend + 1); + size_t len = 1; + int socket_fd; + int ret; + + if (path == NULL) { + TESTPMD_LOG(ERR, "Invalid devargs extension is specified\n"); + return -1; + } + + /* Initialize IPC channel. */ + socket_fd = socket(AF_UNIX, SOCK_SEQPACKET, 0); + if (socket_fd < 0) { + TESTPMD_LOG(ERR, "Failed to create unix socket: %s\n", + strerror(errno)); + return -1; + } + rte_strlcpy(un.sun_path, path, sizeof(un.sun_path)); + if (connect(socket_fd, (struct sockaddr *)&un, sizeof(un)) < 0) { + TESTPMD_LOG(ERR, "Failed to connect %s: %s\n", un.sun_path, + strerror(errno)); + close(socket_fd); + return -1; + } + + /* Send the request message. */ + do { + ret = sendmsg(socket_fd, &msgh, 0); + } while (ret < 0 && errno == EINTR); + if (ret < 0) { + TESTPMD_LOG(ERR, "Failed to send request to (%s): %s\n", path, + strerror(errno)); + close(socket_fd); + return -1; + } + + msgh.msg_iov = &iov; + msgh.msg_iovlen = 1; + msgh.msg_control = control.buf; + msgh.msg_controllen = sizeof(control.buf); + do { + ret = recvmsg(socket_fd, &msgh, 0); + } while (ret < 0); + if (ret != sizeof(int) || (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { + TESTPMD_LOG(ERR, "truncated msg"); + close(socket_fd); + return -1; + } + + /* Translate the FD. */ + cmsg = CMSG_FIRSTHDR(&msgh); + if (cmsg == NULL || cmsg->cmsg_len != CMSG_LEN(sizeof(int)) || + cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) { + TESTPMD_LOG(ERR, "Fail to get FD using SCM_RIGHTS mechanism\n"); + close(socket_fd); + unlink(un.sun_path); + return -1; + } + memcpy(&cmd_fd, CMSG_DATA(cmsg), sizeof(int)); + + TESTPMD_LOG(DEBUG, "Command FD (%d) and PD handle (%d) " + "are successfully imported from remote process\n", + cmd_fd, pd_handle); + + /* Cleanup IPC channel. */ + close(socket_fd); + + /* Calculate the new length of devargs string. */ + len += snprintf(NULL, 0, ",cmd_fd=%d,pd_handle=%d", cmd_fd, pd_handle); + /* Extend the devargs string. */ + snprintf(extend, len, ",cmd_fd=%d,pd_handle=%d", cmd_fd, pd_handle); + + TESTPMD_LOG(DEBUG, "Attach port with extra devargs %s\n", identifier); + return 0; +} + +static bool +is_delimiter_path_spaces(char *extend) +{ + while (*extend != '\0') { + if (*extend != ' ') + return true; + extend++; + } + return false; +} + +/* + * Extend devargs list with "cmd_fd" and "pd_handle" coming from external + * process. It happens only in this format: + * testpmd> mlx5 port attach (identifier) socket= + * all "(identifier) socket=" is in the same string pointed + * by the input parameter 'identifier'. + * + * @param identifier + * Identifier of port attach command line. + */ +static void +mlx5_test_attach_port_extend_devargs(char *identifier) +{ + char *extend; + + if (identifier == NULL) { + fprintf(stderr, "Invalid parameters are specified\n"); + return; + } + + extend = strchr(identifier, ' '); + if (extend != NULL && is_delimiter_path_spaces(extend) && + mlx5_test_extend_devargs(identifier, extend) < 0) { + TESTPMD_LOG(ERR, "Failed to extend devargs for port %s\n", + identifier); + return; + } + + attach_port(identifier); +} +#endif + /* *** SET HOST_SHAPER FOR A PORT *** */ struct cmd_port_host_shaper_result { cmdline_fixed_string_t mlx5; @@ -189,6 +351,56 @@ static cmdline_parse_inst_t mlx5_test_cmd_port_host_shaper = { } }; +#ifndef RTE_EXEC_ENV_WINDOWS +/* *** attach a specified port *** */ +struct mlx5_cmd_operate_attach_port_result { + cmdline_fixed_string_t mlx5; + cmdline_fixed_string_t port; + cmdline_fixed_string_t keyword; + cmdline_multi_string_t identifier; +}; + +static void mlx5_cmd_operate_attach_port_parsed(void *parsed_result, + __rte_unused struct cmdline *cl, + __rte_unused void *data) +{ + struct mlx5_cmd_operate_attach_port_result *res = parsed_result; + + if (!strcmp(res->keyword, "attach")) + mlx5_test_attach_port_extend_devargs(res->identifier); + else + fprintf(stderr, "Unknown parameter\n"); +} + +static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_mlx5 = + TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result, + mlx5, "mlx5"); +static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_port = + TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result, + port, "port"); +static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_keyword = + TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result, + keyword, "attach"); +static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_identifier = + TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result, + identifier, TOKEN_STRING_MULTI); + +static cmdline_parse_inst_t mlx5_cmd_operate_attach_port = { + .f = mlx5_cmd_operate_attach_port_parsed, + .data = NULL, + .help_str = "mlx5 port attach socket=: " + "(identifier: pci address or virtual dev name" + ", path (optional): socket path to get cmd FD and PD handle)", + .tokens = { + (void *)&mlx5_cmd_operate_attach_port_mlx5, + (void *)&mlx5_cmd_operate_attach_port_port, + (void *)&mlx5_cmd_operate_attach_port_keyword, + (void *)&mlx5_cmd_operate_attach_port_identifier, + NULL, + }, +}; +#endif + static struct testpmd_driver_commands mlx5_driver_cmds = { .commands = { { @@ -197,6 +409,14 @@ static struct testpmd_driver_commands mlx5_driver_cmds = { "rate (rate_num):\n" " Set HOST_SHAPER avail_thresh_triggered and rate with port_id\n\n", }, +#ifndef RTE_EXEC_ENV_WINDOWS + { + .ctx = &mlx5_cmd_operate_attach_port, + .help = "mlx5 port attach (ident) socket=(path)\n" + " Attach physical or virtual dev by pci address or virtual device name " + "and add \"cmd_fd\" and \"pd_handle\" devargs before attaching\n\n", + }, +#endif { .ctx = NULL, }, -- 2.20.1