net/mlx4: spawn rdma-core dependency plug-in

When mlx4 is not compiled directly as an independent shared object (e.g.
CONFIG_RTE_BUILD_SHARED_LIB not enabled for performance reasons), DPDK
applications inherit its dependencies on libibverbs and libmlx4 through
rte.app.mk.

This is an issue both when DPDK is delivered as a binary package (Linux
distributions) and for end users because rdma-core then propagates as a
mandatory dependency for everything.

Application writers relying on binary DPDK packages are not necessarily
aware of this fact and may end up delivering packages with broken
dependencies.

This patch therefore introduces an intermediate internal plug-in
hard-linked with rdma-core (to preserve symbol versioning) loaded by the
PMD through dlopen(), so that a missing rdma-core does not cause unresolved
symbols, allowing applications to start normally.

Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>

diff --git a/config/common_base b/config/common_base
index 208b40b442e75e1d79324c1ef58f2d509350826c..b0a2d0062cbecac4c6246aeb3050afa323fe481b 100644 (file)
--- a/config/common_base
+++ b/config/common_base
@@ -298,6 +298,7 @@ CONFIG_RTE_LIBRTE_AVF_16BYTE_RX_DESC=n
 #
 CONFIG_RTE_LIBRTE_MLX4_PMD=n
 CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
+CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS=n
 CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
 
 #

diff --git a/doc/guides/nics/mlx4.rst b/doc/guides/nics/mlx4.rst
index cab45dfe84b355813bde03886f5335fa17a472c4..88161781c091d32f61ffdf05f05565ee58ffd6d0 100644 (file)
--- a/doc/guides/nics/mlx4.rst
+++ b/doc/guides/nics/mlx4.rst
@@ -86,6 +86,19 @@ These options can be modified in the ``.config`` file.
 
   Toggle compilation of librte_pmd_mlx4 itself.
 
+- ``CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS`` (default **n**)
+
+  Build PMD with additional code to make it loadable without hard
+  dependencies on **libibverbs** nor **libmlx4**, which may not be installed
+  on the target system.
+
+  In this mode, their presence is still required for it to run properly,
+  however their absence won't prevent a DPDK application from starting (with
+  ``CONFIG_RTE_BUILD_SHARED_LIB`` disabled) and they won't show up as
+  missing with ``ldd(1)``.
+
+  This option has no performance impact.
+
 - ``CONFIG_RTE_LIBRTE_MLX4_DEBUG`` (default **n**)
 
   Toggle debugging code and stricter compilation flags. Enabling this option

diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index 1d33c38ed1fc37b94ae0632ae1c385f293c3b094..c004ac71c4467ffb09e5804d5734098f631e880c 100644 (file)
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -33,12 +33,15 @@ include $(RTE_SDK)/mk/rte.vars.mk
 
 # Library name.
 LIB = librte_pmd_mlx4.a
+LIB_GLUE = librte_pmd_mlx4_glue.so
 
 # Sources.
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_flow.c
+ifneq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y)
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_glue.c
+endif
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_intr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c
@@ -46,6 +49,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_utils.c
 
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y)
+INSTALL-$(CONFIG_RTE_LIBRTE_MLX4_PMD)-lib += $(LIB_GLUE)
+endif
+
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -55,7 +62,13 @@ CFLAGS += -D_BSD_SOURCE
 CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y)
+CFLAGS += -DMLX4_GLUE='"$(LIB_GLUE)"'
+CFLAGS_mlx4_glue.o += -fPIC
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx4
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -109,7 +122,23 @@ mlx4_autoconf.h: mlx4_autoconf.h.new
 
 $(SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD):.c=.o): mlx4_autoconf.h
 
+# Generate dependency plug-in for rdma-core when the PMD must not be linked
+# directly, so that applications do not inherit this dependency.
+
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y)
+
+$(LIB): $(LIB_GLUE)
+
+$(LIB_GLUE): mlx4_glue.o
+       $Q $(LD) $(LDFLAGS) $(EXTRA_LDFLAGS) \
+               -s -shared -o $@ $< -libverbs -lmlx4
+
+mlx4_glue.o: mlx4_autoconf.h
+
+endif
+
 clean_mlx4: FORCE
        $Q rm -f -- mlx4_autoconf.h mlx4_autoconf.h.new
+       $Q rm -f -- mlx4_glue.o $(LIB_GLUE)
 
 clean: clean_mlx4

diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index feabbb88281c9eb94a34597f46055d5d8d4c07e4..50a55ee5272f91d6cc4377e4d28c25055d98eb54 100644 (file)
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -37,6 +37,7 @@
  */
 
 #include <assert.h>
+#include <dlfcn.h>
 #include <errno.h>
 #include <inttypes.h>
 #include <stddef.h>
@@ -44,6 +45,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 
 /* Verbs headers do not support -pedantic. */
 #ifdef PEDANTIC
@@ -55,6 +57,7 @@
 #endif
 
 #include <rte_common.h>
+#include <rte_config.h>
 #include <rte_dev.h>
 #include <rte_errno.h>
 #include <rte_ethdev_driver.h>
@@ -730,6 +733,47 @@ static struct rte_pci_driver mlx4_driver = {
                     RTE_PCI_DRV_INTR_RMV,
 };
 
+#ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS
+
+/**
+ * Initialization routine for run-time dependency on rdma-core.
+ */
+static int
+mlx4_glue_init(void)
+{
+       void *handle = NULL;
+       void **sym;
+       const char *dlmsg;
+
+       handle = dlopen(MLX4_GLUE, RTLD_LAZY);
+       if (!handle) {
+               rte_errno = EINVAL;
+               dlmsg = dlerror();
+               if (dlmsg)
+                       WARN("cannot load glue library: %s", dlmsg);
+               goto glue_error;
+       }
+       sym = dlsym(handle, "mlx4_glue");
+       if (!sym || !*sym) {
+               rte_errno = EINVAL;
+               dlmsg = dlerror();
+               if (dlmsg)
+                       ERROR("cannot resolve glue symbol: %s", dlmsg);
+               goto glue_error;
+       }
+       mlx4_glue = *sym;
+       return 0;
+glue_error:
+       if (handle)
+               dlclose(handle);
+       WARN("cannot initialize PMD due to missing run-time"
+            " dependency on rdma-core libraries (libibverbs,"
+            " libmlx4)");
+       return -rte_errno;
+}
+
+#endif
+
 /**
  * Driver initialization routine.
  */
@@ -750,6 +794,11 @@ rte_mlx4_pmd_init(void)
         * using this PMD, which is not supported in forked processes.
         */
        setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+#ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS
+       if (mlx4_glue_init())
+               return;
+       assert(mlx4_glue);
+#endif
        mlx4_glue->fork_init();
        rte_pci_register(&mlx4_driver);
 }

diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index e683da4c9d71da0519e438b20b185f1d96ccabf1..57ce8f752556eddeb1485f6951eec78539f778d7 100644 (file)
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -144,7 +144,11 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -ldl
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp