]> git.droids-corp.org - dpdk.git/commitdiff
common/mlx5: support auxiliary bus
authorXueming Li <xuemingl@nvidia.com>
Wed, 21 Jul 2021 14:37:31 +0000 (22:37 +0800)
committerThomas Monjalon <thomas@monjalon.net>
Wed, 21 Jul 2021 22:11:14 +0000 (00:11 +0200)
Add auxiliary bus support for Sub-Function.

As a limitation of current driver, NUMA node of device is detected
from PCI bus of device sysfs symbol link.
It will be removed once NUMA node file will be available in sysfs.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
drivers/common/mlx5/linux/meson.build
drivers/common/mlx5/linux/mlx5_common_auxiliary.c [new file with mode: 0644]
drivers/common/mlx5/linux/mlx5_common_verbs.c
drivers/common/mlx5/meson.build
drivers/common/mlx5/mlx5_common.c
drivers/common/mlx5/mlx5_common.h
drivers/common/mlx5/mlx5_common_private.h
drivers/common/mlx5/version.map

index 686df2690998d83b034552b811cc78344fd3aa81..6abf27ce638c87721e720178adb52dd7fcef1674 100644 (file)
@@ -46,12 +46,15 @@ if static_ibverbs
 endif
 
 sources += files('mlx5_nl.c')
+sources += files('mlx5_common_auxiliary.c')
 sources += files('mlx5_common_os.c')
 sources += files('mlx5_common_verbs.c')
 if not dlopen_ibverbs
     sources += files('mlx5_glue.c')
 endif
 
+deps += ['bus_auxiliary']
+
 # To maintain the compatibility with the make build system
 # mlx5_autoconf.h file is still generated.
 # input array for meson member search:
diff --git a/drivers/common/mlx5/linux/mlx5_common_auxiliary.c b/drivers/common/mlx5/linux/mlx5_common_auxiliary.c
new file mode 100644 (file)
index 0000000..7b4efed
--- /dev/null
@@ -0,0 +1,175 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies Ltd
+ */
+
+#include <stdlib.h>
+#include <dirent.h>
+#include <libgen.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include <rte_bus_auxiliary.h>
+#include <rte_common.h>
+#include "eal_filesystem.h"
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+#define MLX5_AUXILIARY_PREFIX "mlx5_core.sf."
+
+int
+mlx5_auxiliary_get_child_name(const char *dev, const char *node,
+                             char *child, size_t size)
+{
+       DIR *dir;
+       struct dirent *dent;
+       MKSTR(path, "%s/%s%s", AUXILIARY_SYSFS_PATH, dev, node);
+
+       dir = opendir(path);
+       if (dir == NULL) {
+               rte_errno = errno;
+               return -rte_errno;
+       }
+       /* Get the first file name. */
+       while ((dent = readdir(dir)) != NULL) {
+               if (dent->d_name[0] != '.')
+                       break;
+       }
+       closedir(dir);
+       if (dent == NULL) {
+               rte_errno = ENOENT;
+               return -rte_errno;
+       }
+       if (rte_strscpy(child, dent->d_name, size) < 0)
+               return -rte_errno;
+       return 0;
+}
+
+static int
+mlx5_auxiliary_get_pci_path(const struct rte_auxiliary_device *dev,
+                           char *sysfs_pci, size_t size)
+{
+       char sysfs_real[PATH_MAX] = { 0 };
+       MKSTR(sysfs_aux, "%s/%s", AUXILIARY_SYSFS_PATH, dev->name);
+       char *dir;
+
+       if (realpath(sysfs_aux, sysfs_real) == NULL) {
+               rte_errno = errno;
+               return -rte_errno;
+       }
+       dir = dirname(sysfs_real);
+       if (dir == NULL) {
+               rte_errno = errno;
+               return -rte_errno;
+       }
+       if (rte_strscpy(sysfs_pci, dir, size) < 0)
+               return -rte_errno;
+       return 0;
+}
+
+static int
+mlx5_auxiliary_get_numa(const struct rte_auxiliary_device *dev)
+{
+       unsigned long numa;
+       char numa_path[PATH_MAX];
+
+       if (mlx5_auxiliary_get_pci_path(dev, numa_path, sizeof(numa_path)) != 0)
+               return SOCKET_ID_ANY;
+       if (strcat(numa_path, "/numa_node") == NULL) {
+               rte_errno = ENAMETOOLONG;
+               return SOCKET_ID_ANY;
+       }
+       if (eal_parse_sysfs_value(numa_path, &numa) != 0) {
+               rte_errno = EINVAL;
+               return SOCKET_ID_ANY;
+       }
+       return (int)numa;
+}
+
+struct ibv_device *
+mlx5_get_aux_ibv_device(const struct rte_auxiliary_device *dev)
+{
+       int n;
+       char ib_name[64] = { 0 };
+       struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
+       struct ibv_device *ibv_match = NULL;
+
+       if (!ibv_list) {
+               rte_errno = ENOSYS;
+               return NULL;
+       }
+       if (mlx5_auxiliary_get_child_name(dev->name, "/infiniband",
+                                         ib_name, sizeof(ib_name)) != 0)
+               goto out;
+       while (n-- > 0) {
+               if (strcmp(ibv_list[n]->name, ib_name) != 0)
+                       continue;
+               ibv_match = ibv_list[n];
+               break;
+       }
+       if (ibv_match == NULL)
+               rte_errno = ENOENT;
+out:
+       mlx5_glue->free_device_list(ibv_list);
+       return ibv_match;
+}
+
+static bool
+mlx5_common_auxiliary_match(const char *name)
+{
+       return strncmp(name, MLX5_AUXILIARY_PREFIX,
+                      strlen(MLX5_AUXILIARY_PREFIX)) == 0;
+}
+
+static int
+mlx5_common_auxiliary_probe(struct rte_auxiliary_driver *drv __rte_unused,
+                           struct rte_auxiliary_device *dev)
+{
+       dev->device.numa_node = mlx5_auxiliary_get_numa(dev);
+       return mlx5_common_dev_probe(&dev->device);
+}
+
+static int
+mlx5_common_auxiliary_remove(struct rte_auxiliary_device *auxiliary_dev)
+{
+       return mlx5_common_dev_remove(&auxiliary_dev->device);
+}
+
+static int
+mlx5_common_auxiliary_dma_map(struct rte_auxiliary_device *auxiliary_dev,
+                             void *addr, uint64_t iova, size_t len)
+{
+       return mlx5_common_dev_dma_map(&auxiliary_dev->device, addr, iova, len);
+}
+
+static int
+mlx5_common_auxiliary_dma_unmap(struct rte_auxiliary_device *auxiliary_dev,
+                               void *addr, uint64_t iova, size_t len)
+{
+       return mlx5_common_dev_dma_unmap(&auxiliary_dev->device, addr, iova,
+                                        len);
+}
+
+static struct rte_auxiliary_driver mlx5_auxiliary_driver = {
+       .driver = {
+                  .name = MLX5_AUXILIARY_DRIVER_NAME,
+       },
+       .match = mlx5_common_auxiliary_match,
+       .probe = mlx5_common_auxiliary_probe,
+       .remove = mlx5_common_auxiliary_remove,
+       .dma_map = mlx5_common_auxiliary_dma_map,
+       .dma_unmap = mlx5_common_auxiliary_dma_unmap,
+};
+
+void mlx5_common_auxiliary_init(void)
+{
+       if (mlx5_auxiliary_driver.bus == NULL)
+               rte_auxiliary_register(&mlx5_auxiliary_driver);
+}
+
+RTE_FINI(mlx5_common_auxiliary_driver_finish)
+{
+       if (mlx5_auxiliary_driver.bus != NULL)
+               rte_auxiliary_unregister(&mlx5_auxiliary_driver);
+}
index 6a6ab7a7a21ec132946e445ec419aa0235363de2..9080bd3e8738bb59b43c6e857d8e95187df610f6 100644 (file)
@@ -12,6 +12,7 @@
 
 #include <rte_errno.h>
 #include <rte_bus_pci.h>
+#include <rte_bus_auxiliary.h>
 
 #include "mlx5_common_utils.h"
 #include "mlx5_common_log.h"
 struct ibv_device *
 mlx5_os_get_ibv_dev(const struct rte_device *dev)
 {
-       struct ibv_device *ibv = NULL;
+       struct ibv_device *ibv;
 
        if (mlx5_dev_is_pci(dev))
                ibv = mlx5_os_get_ibv_device(&RTE_DEV_TO_PCI_CONST(dev)->addr);
+       else
+               ibv = mlx5_get_aux_ibv_device(RTE_DEV_TO_AUXILIARY_CONST(dev));
        if (ibv == NULL) {
                rte_errno = ENODEV;
                DRV_LOG(ERR, "Verbs device not found: %s", dev->name);
index fa2b8b9834767fc1ec35a66e27e02a173fa5a2e3..6ddbde7e8f332288df4f4be37669700efc3ea7f5 100644 (file)
@@ -7,7 +7,7 @@ if not (is_linux or (is_windows and is_ms_linker))
     subdir_done()
 endif
 
-deps += ['hash', 'pci', 'bus_pci', 'net', 'eal', 'kvargs']
+deps += ['hash', 'pci', 'bus_pci', 'bus_auxiliary', 'net', 'eal', 'kvargs']
 sources += files(
         'mlx5_devx_cmds.c',
         'mlx5_common.c',
index 97d3e3e60e5f71b8d4eb54b054f9cdfbc5293def..94f858a9bd04887a31b725b4de468cc5abf31221 100644 (file)
@@ -398,6 +398,9 @@ mlx5_class_driver_register(struct mlx5_class_driver *driver)
 static void mlx5_common_driver_init(void)
 {
        mlx5_common_pci_init();
+#ifdef RTE_EXEC_ENV_LINUX
+       mlx5_common_auxiliary_init();
+#endif
 }
 
 static bool mlx5_common_initialized;
index 83b6cfc214d26b4b1c5df9af1e7c9bd1c694ccfc..e073039f43a1c066349ec4ba2f03d27f598db3af 100644 (file)
@@ -24,6 +24,7 @@
 
 /* Reported driver name. */
 #define MLX5_PCI_DRIVER_NAME "mlx5_pci"
+#define MLX5_AUXILIARY_DRIVER_NAME "mlx5_auxiliary"
 
 /* Bit-field manipulation. */
 #define BITFIELD_DECLARE(bf, type, size) \
@@ -109,6 +110,7 @@ pmd_drv_log_basename(const char *s)
        int mkstr_size_##name = snprintf(NULL, 0, "" __VA_ARGS__); \
        char name[mkstr_size_##name + 1]; \
        \
+       memset(name, 0, mkstr_size_##name + 1); \
        snprintf(name, sizeof(name), "" __VA_ARGS__)
 
 enum {
@@ -235,6 +237,9 @@ int mlx5_dev_to_pci_addr(const char *dev_path, struct rte_pci_addr *pci_addr);
 __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
+__rte_internal
+int mlx5_auxiliary_get_child_name(const char *dev, const char *node,
+                                 char *child, size_t size);
 
 enum mlx5_class {
        MLX5_CLASS_INVALID,
index 93a60a0bb3c1077f99cf316fcfe97672adac3eca..5168cfdcca0f7f27230ab2f01da3913eaa4514d1 100644 (file)
@@ -6,6 +6,7 @@
 #define MLX5_COMMON_PRIVATE_H
 
 #include <rte_pci.h>
+#include <rte_bus_auxiliary.h>
 
 #include "mlx5_common.h"
 
@@ -34,6 +35,11 @@ void mlx5_common_driver_on_register_pci(struct mlx5_class_driver *driver);
 bool mlx5_dev_pci_match(const struct mlx5_class_driver *drv,
                        const struct rte_device *dev);
 
+/* Common auxiliary bus driver: */
+void mlx5_common_auxiliary_init(void);
+struct ibv_device *mlx5_get_aux_ibv_device(
+               const struct rte_auxiliary_device *dev);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */
index cc8b34da1b55b5d0333f6711af896b91db257fc1..511077146754419ff5295b3c9492e010a5d6130c 100644 (file)
@@ -3,6 +3,8 @@ INTERNAL {
 
        haswell_broadwell_cpu;
 
+       mlx5_auxiliary_get_child_name; # WINDOWS_NO_EXPORT
+
        mlx5_class_driver_register;
 
        mlx5_common_init;