net/mlx5: add test for remote PD and CTX
authorMichael Baum <michaelba@nvidia.com>
Tue, 28 Jun 2022 14:58:40 +0000 (17:58 +0300)
committerThomas Monjalon <thomas@monjalon.net>
Tue, 5 Jul 2022 18:01:33 +0000 (20:01 +0200)
Add mlx5 internal option in testpmd similar to run-time function
"port attach" which adds another parameter named "socket" for attaching
port and add 2 devargs before.

The arguments are "cmd_fd" and "pd_handle" using to import device
created out of PMD. Testpmd application import it using IPC, and updates
the devargs list before attaching.

These arguments were added in
the commit 9d936f4f1a5e ("common/mlx5: support remote PD and CTX")

The syntax is:

  testpmd> mlx5 port attach (identifier) socket=(path)

Where "path" is the IPC socket path agreed on the remote process.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Reviewed-by: Thomas Monjalon <thomas@monjalon.net>
Acked-by: Matan Azrad <matan@nvidia.com>
doc/guides/nics/mlx5.rst
doc/guides/testpmd_app_ug/testpmd_funcs.rst
drivers/net/mlx5/mlx5_testpmd.c

index bc2bd2c..cd3a613 100644 (file)
@@ -1793,3 +1793,50 @@ and disables ``avail_thresh_triggered``.
 .. code-block:: console
 
    testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 50
+
+
+Testpmd
+-------
+
+port attach with socket path
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is possible to allocate a port with ``libibverbs`` from external application.
+For importing the external port with extra device arguments,
+there is a specific testpmd command
+similar to :ref:`port attach command <port_attach>`::
+
+   testpmd> mlx5 port attach (identifier) socket=(path)
+
+where:
+
+* ``identifier``: device identifier with optional parameters
+  as same as :ref:`port attach command <port_attach>`.
+* ``path``: path to IPC server socket created by the external application.
+
+This command performs:
+
+#. Open IPC client socket using the given path, and connect it.
+
+#. Import ibverbs context and ibverbs protection domain.
+
+#. Add two device arguments for context (``cmd_fd``)
+   and protection domain (``pd_handle``) to the device identifier.
+   See :ref:`mlx5 driver options <mlx5_common_driver_options>` for more
+   information about these device arguments.
+
+#. Call the regular ``port attach`` function with updated identifier.
+
+For example, to attach a port whose PCI address is ``0000:0a:00.0``
+and its socket path is ``/var/run/import_ipc_socket``:
+
+.. code-block:: console
+
+   testpmd> mlx5 port attach 0000:0a:00.0 socket=/var/run/import_ipc_socket
+   testpmd: MLX5 socket path is /var/run/import_ipc_socket
+   testpmd: Attach port with extra devargs 0000:0a:00.0,cmd_fd=40,pd_handle=1
+   Attaching a new port...
+   EAL: Probe PCI driver: mlx5_pci (15b3:101d) device: 0000:0a:00.0 (socket 0)
+   Port 0 is attached. Now total ports is 1
+   Done
+
index b0bb912..330e344 100644 (file)
@@ -1882,6 +1882,8 @@ The following sections show functions for configuring ports.
 
    Port configuration changes only become active when forwarding is started/restarted.
 
+.. _port_attach:
+
 port attach
 ~~~~~~~~~~~
 
index 4f98264..463ee8e 100644 (file)
@@ -6,6 +6,11 @@
 #include <stdint.h>
 #include <string.h>
 #include <stdlib.h>
+#include <unistd.h>
+#ifndef RTE_EXEC_ENV_WINDOWS
+#include <sys/socket.h>
+#include <sys/un.h>
+#endif
 
 #include <rte_prefetch.h>
 #include <rte_common.h>
@@ -14,6 +19,7 @@
 #include <rte_alarm.h>
 #include <rte_pmd_mlx5.h>
 #include <rte_ethdev.h>
+
 #include "mlx5_testpmd.h"
 #include "testpmd.h"
 
@@ -111,6 +117,162 @@ mlx5_test_set_port_host_shaper(uint16_t port_id, uint16_t avail_thresh_triggered
        return 0;
 }
 
+#ifndef RTE_EXEC_ENV_WINDOWS
+static const char*
+mlx5_test_get_socket_path(char *extend)
+{
+       if (strstr(extend, "socket=") == extend) {
+               const char *socket_path = strchr(extend, '=') + 1;
+
+               TESTPMD_LOG(DEBUG, "MLX5 socket path is %s\n", socket_path);
+               return socket_path;
+       }
+
+       TESTPMD_LOG(ERR, "Failed to extract a valid socket path from %s\n",
+                   extend);
+       return NULL;
+}
+
+static int
+mlx5_test_extend_devargs(char *identifier, char *extend)
+{
+       struct sockaddr_un un = {
+               .sun_family = AF_UNIX,
+       };
+       int cmd_fd;
+       int pd_handle;
+       struct iovec iov = {
+               .iov_base = &pd_handle,
+               .iov_len = sizeof(int),
+       };
+       union {
+               char buf[CMSG_SPACE(sizeof(int))];
+               struct cmsghdr align;
+       } control;
+       struct msghdr msgh = {
+               .msg_iov = NULL,
+               .msg_iovlen = 0,
+       };
+       struct cmsghdr *cmsg;
+       const char *path = mlx5_test_get_socket_path(extend + 1);
+       size_t len = 1;
+       int socket_fd;
+       int ret;
+
+       if (path == NULL) {
+               TESTPMD_LOG(ERR, "Invalid devargs extension is specified\n");
+               return -1;
+       }
+
+       /* Initialize IPC channel. */
+       socket_fd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
+       if (socket_fd < 0) {
+               TESTPMD_LOG(ERR, "Failed to create unix socket: %s\n",
+                           strerror(errno));
+               return -1;
+       }
+       rte_strlcpy(un.sun_path, path, sizeof(un.sun_path));
+       if (connect(socket_fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
+               TESTPMD_LOG(ERR, "Failed to connect %s: %s\n", un.sun_path,
+                           strerror(errno));
+               close(socket_fd);
+               return -1;
+       }
+
+       /* Send the request message. */
+       do {
+               ret = sendmsg(socket_fd, &msgh, 0);
+       } while (ret < 0 && errno == EINTR);
+       if (ret < 0) {
+               TESTPMD_LOG(ERR, "Failed to send request to (%s): %s\n", path,
+                           strerror(errno));
+               close(socket_fd);
+               return -1;
+       }
+
+       msgh.msg_iov = &iov;
+       msgh.msg_iovlen = 1;
+       msgh.msg_control = control.buf;
+       msgh.msg_controllen = sizeof(control.buf);
+       do {
+               ret = recvmsg(socket_fd, &msgh, 0);
+       } while (ret < 0);
+       if (ret != sizeof(int) || (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
+               TESTPMD_LOG(ERR, "truncated msg");
+               close(socket_fd);
+               return -1;
+       }
+
+       /* Translate the FD. */
+       cmsg = CMSG_FIRSTHDR(&msgh);
+       if (cmsg == NULL || cmsg->cmsg_len != CMSG_LEN(sizeof(int)) ||
+           cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {
+               TESTPMD_LOG(ERR, "Fail to get FD using SCM_RIGHTS mechanism\n");
+               close(socket_fd);
+               unlink(un.sun_path);
+               return -1;
+       }
+       memcpy(&cmd_fd, CMSG_DATA(cmsg), sizeof(int));
+
+       TESTPMD_LOG(DEBUG, "Command FD (%d) and PD handle (%d) "
+                   "are successfully imported from remote process\n",
+                   cmd_fd, pd_handle);
+
+       /* Cleanup IPC channel. */
+       close(socket_fd);
+
+       /* Calculate the new length of devargs string. */
+       len += snprintf(NULL, 0, ",cmd_fd=%d,pd_handle=%d", cmd_fd, pd_handle);
+       /* Extend the devargs string. */
+       snprintf(extend, len, ",cmd_fd=%d,pd_handle=%d", cmd_fd, pd_handle);
+
+       TESTPMD_LOG(DEBUG, "Attach port with extra devargs %s\n", identifier);
+       return 0;
+}
+
+static bool
+is_delimiter_path_spaces(char *extend)
+{
+       while (*extend != '\0') {
+               if (*extend != ' ')
+                       return true;
+               extend++;
+       }
+       return false;
+}
+
+/*
+ * Extend devargs list with "cmd_fd" and "pd_handle" coming from external
+ * process. It happens only in this format:
+ *  testpmd> mlx5 port attach (identifier) socket=<socket path>
+ * all "(identifier) socket=<socket path>" is in the same string pointed
+ * by the input parameter 'identifier'.
+ *
+ * @param identifier
+ *   Identifier of port attach command line.
+ */
+static void
+mlx5_test_attach_port_extend_devargs(char *identifier)
+{
+       char *extend;
+
+       if (identifier == NULL) {
+               fprintf(stderr, "Invalid parameters are specified\n");
+               return;
+       }
+
+       extend = strchr(identifier, ' ');
+       if (extend != NULL && is_delimiter_path_spaces(extend) &&
+           mlx5_test_extend_devargs(identifier, extend) < 0) {
+               TESTPMD_LOG(ERR, "Failed to extend devargs for port %s\n",
+                           identifier);
+               return;
+       }
+
+       attach_port(identifier);
+}
+#endif
+
 /* *** SET HOST_SHAPER FOR A PORT *** */
 struct cmd_port_host_shaper_result {
        cmdline_fixed_string_t mlx5;
@@ -189,6 +351,56 @@ static cmdline_parse_inst_t mlx5_test_cmd_port_host_shaper = {
        }
 };
 
+#ifndef RTE_EXEC_ENV_WINDOWS
+/* *** attach a specified port *** */
+struct mlx5_cmd_operate_attach_port_result {
+       cmdline_fixed_string_t mlx5;
+       cmdline_fixed_string_t port;
+       cmdline_fixed_string_t keyword;
+       cmdline_multi_string_t identifier;
+};
+
+static void mlx5_cmd_operate_attach_port_parsed(void *parsed_result,
+                                               __rte_unused struct cmdline *cl,
+                                               __rte_unused void *data)
+{
+       struct mlx5_cmd_operate_attach_port_result *res = parsed_result;
+
+       if (!strcmp(res->keyword, "attach"))
+               mlx5_test_attach_port_extend_devargs(res->identifier);
+       else
+               fprintf(stderr, "Unknown parameter\n");
+}
+
+static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_mlx5 =
+       TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result,
+                                mlx5, "mlx5");
+static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_port =
+       TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result,
+                                port, "port");
+static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_keyword =
+       TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result,
+                                keyword, "attach");
+static cmdline_parse_token_string_t mlx5_cmd_operate_attach_port_identifier =
+       TOKEN_STRING_INITIALIZER(struct mlx5_cmd_operate_attach_port_result,
+                                identifier, TOKEN_STRING_MULTI);
+
+static cmdline_parse_inst_t mlx5_cmd_operate_attach_port = {
+       .f = mlx5_cmd_operate_attach_port_parsed,
+       .data = NULL,
+       .help_str = "mlx5 port attach <identifier> socket=<path>: "
+               "(identifier: pci address or virtual dev name"
+               ", path (optional): socket path to get cmd FD and PD handle)",
+       .tokens = {
+               (void *)&mlx5_cmd_operate_attach_port_mlx5,
+               (void *)&mlx5_cmd_operate_attach_port_port,
+               (void *)&mlx5_cmd_operate_attach_port_keyword,
+               (void *)&mlx5_cmd_operate_attach_port_identifier,
+               NULL,
+       },
+};
+#endif
+
 static struct testpmd_driver_commands mlx5_driver_cmds = {
        .commands = {
                {
@@ -197,6 +409,14 @@ static struct testpmd_driver_commands mlx5_driver_cmds = {
                                "rate (rate_num):\n"
                                "    Set HOST_SHAPER avail_thresh_triggered and rate with port_id\n\n",
                },
+#ifndef RTE_EXEC_ENV_WINDOWS
+               {
+                       .ctx = &mlx5_cmd_operate_attach_port,
+                       .help = "mlx5 port attach (ident) socket=(path)\n"
+                               "    Attach physical or virtual dev by pci address or virtual device name "
+                               "and add \"cmd_fd\" and \"pd_handle\" devargs before attaching\n\n",
+               },
+#endif
                {
                        .ctx = NULL,
                },