vfio: export some internal functions

This patch moves some of the internal vfio functions from
eal_vfio.h to rte_vfio.h for common uses with "rte_" prefix.

This patch also change the FSLMC bus usages from the internal
VFIO functions to external ones with "rte_" prefix

Signed-off-by: Hemant Agrawal <hemant.agrawal@nxp.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile
index 93870ba..3aa34e2 100644 (file)
--- a/drivers/bus/fslmc/Makefile
+++ b/drivers/bus/fslmc/Makefile
@@ -16,7 +16,6 @@ CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc
 CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc
 CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include
-CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
 CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev

diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 625fa7c..4036e82 100644 (file)
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -91,7 +91,8 @@ fslmc_get_container_group(int *groupid)
        }
 
        /* get group number */
-       ret = vfio_get_group_no(SYSFS_FSL_MC_DEVICES, g_container, groupid);
+       ret = rte_vfio_get_group_num(SYSFS_FSL_MC_DEVICES,
+                                    g_container, groupid);
        if (ret <= 0) {
                DPAA2_BUS_ERR("Unable to find %s IOMMU group", g_container);
                return -1;
@@ -124,7 +125,7 @@ vfio_connect_container(void)
        }
 
        /* Opens main vfio file descriptor which represents the "container" */
-       fd = vfio_get_container_fd();
+       fd = rte_vfio_get_container_fd();
        if (fd < 0) {
                DPAA2_BUS_ERR("Failed to open VFIO container");
                return -errno;
@@ -726,7 +727,7 @@ fslmc_vfio_setup_group(void)
        }
 
        /* Get the actual group fd */
-       ret = vfio_get_group_fd(groupid);
+       ret = rte_vfio_get_group_fd(groupid);
        if (ret < 0)
                return ret;
        vfio_group.fd = ret;

diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index e8fb344..9e2c4fe 100644 (file)
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -10,8 +10,6 @@
 
 #include <rte_vfio.h>
 
-#include "eal_vfio.h"
-
 #define DPAA2_MC_DPNI_DEVID    7
 #define DPAA2_MC_DPSECI_DEVID  3
 #define DPAA2_MC_DPCON_DEVID   5

diff --git a/drivers/bus/fslmc/meson.build b/drivers/bus/fslmc/meson.build
index e94340e..78f9d92 100644 (file)
--- a/drivers/bus/fslmc/meson.build
+++ b/drivers/bus/fslmc/meson.build
@@ -22,6 +22,5 @@ sources = files('fslmc_bus.c',
 
 allow_experimental_apis = true
 
-includes += include_directories('../../../lib/librte_eal/linuxapp/eal')
 includes += include_directories('mc', 'qbman/include', 'portal')
 cflags += ['-D_GNU_SOURCE']

diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index 727adc5..bfbec0d 100644 (file)
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -818,3 +818,23 @@ rte_vfio_dma_unmap(uint64_t __rte_unused vaddr, uint64_t __rte_unused iova,
 {
        return -1;
 }
+
+int __rte_experimental
+rte_vfio_get_group_num(__rte_unused const char *sysfs_base,
+                      __rte_unused const char *dev_addr,
+                      __rte_unused int *iommu_group_num)
+{
+       return -1;
+}
+
+int  __rte_experimental
+rte_vfio_get_container_fd(void)
+{
+       return -1;
+}
+
+int  __rte_experimental
+rte_vfio_get_group_fd(__rte_unused int iommu_group_num)
+{
+       return -1;
+}

diff --git a/lib/librte_eal/common/include/rte_vfio.h b/lib/librte_eal/common/include/rte_vfio.h
index f363cac..c4a2e60 100644 (file)
--- a/lib/librte_eal/common/include/rte_vfio.h
+++ b/lib/librte_eal/common/include/rte_vfio.h
@@ -37,6 +37,13 @@
 extern "C" {
 #endif
 
+/* NOIOMMU is defined from kernel version 4.5 onwards */
+#ifdef VFIO_NOIOMMU_IOMMU
+#define RTE_VFIO_NOIOMMU VFIO_NOIOMMU_IOMMU
+#else
+#define RTE_VFIO_NOIOMMU 8
+#endif
+
 /**
  * Setup vfio_cfg for the device identified by its address.
  * It discovers the configured I/O MMU groups or sets a new one for the device.
@@ -128,7 +135,19 @@ int rte_vfio_is_enabled(const char *modname);
  */
 int rte_vfio_noiommu_is_enabled(void);
 
-/* remove group fd from internal VFIO group fd array */
+/**
+ * Remove group fd from internal VFIO group fd array/
+ *
+ * This function is only relevant to linux and will return
+ * an error on BSD.
+ *
+ * @param vfio_group_fd
+ *   VFIO Grouup FD.
+ *
+ * @return
+ *   0 on success.
+ *   <0 on failure.
+ */
 int
 rte_vfio_clear_group(int vfio_group_fd);
 
@@ -170,8 +189,61 @@ rte_vfio_dma_map(uint64_t vaddr, uint64_t iova, uint64_t len);
  *   0 if success.
  *   -1 on error.
  */
+
 int __rte_experimental
 rte_vfio_dma_unmap(uint64_t vaddr, uint64_t iova, uint64_t len);
+/**
+ * Parse IOMMU group number for a device
+ *
+ * This function is only relevant to linux and will return
+ * an error on BSD.
+ *
+ * @param sysfs_base
+ *   sysfs path prefix.
+ *
+ * @param dev_addr
+ *   device location.
+ *
+ * @param iommu_group_num
+ *   iommu group number
+ *
+ * @return
+ *  >0 on success
+ *   0 for non-existent group or VFIO
+ *  <0 for errors
+ */
+int __rte_experimental
+rte_vfio_get_group_num(const char *sysfs_base,
+                     const char *dev_addr, int *iommu_group_num);
+
+/**
+ * Open VFIO container fd or get an existing one
+ *
+ * This function is only relevant to linux and will return
+ * an error on BSD.
+ *
+ * @return
+ *  > 0 container fd
+ *  < 0 for errors
+ */
+int __rte_experimental
+rte_vfio_get_container_fd(void);
+
+/**
+ * Open VFIO group fd or get an existing one
+ *
+ * This function is only relevant to linux and will return
+ * an error on BSD.
+ *
+ * @param iommu_group_num
+ *   iommu group number
+ *
+ * @return
+ *  > 0 group fd
+ *  < 0 for errors
+ */
+int __rte_experimental
+rte_vfio_get_group_fd(int iommu_group_num);
 
 #ifdef __cplusplus
 }

diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c b/lib/librte_eal/linuxapp/eal/eal_vfio.c
index 589d7d4..984b1a1 100644 (file)
--- a/lib/librte_eal/linuxapp/eal/eal_vfio.c
+++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c
@@ -284,7 +284,7 @@ compact_user_maps(void)
 }
 
 int
-vfio_get_group_fd(int iommu_group_no)
+rte_vfio_get_group_fd(int iommu_group_num)
 {
        int i;
        int vfio_group_fd;
@@ -293,7 +293,7 @@ vfio_get_group_fd(int iommu_group_no)
 
        /* check if we already have the group descriptor open */
        for (i = 0; i < VFIO_MAX_GROUPS; i++)
-               if (vfio_cfg.vfio_groups[i].group_no == iommu_group_no)
+               if (vfio_cfg.vfio_groups[i].group_num == iommu_group_num)
                        return vfio_cfg.vfio_groups[i].fd;
 
        /* Lets see first if there is room for a new group */
@@ -304,7 +304,7 @@ vfio_get_group_fd(int iommu_group_no)
 
        /* Now lets get an index for the new group */
        for (i = 0; i < VFIO_MAX_GROUPS; i++)
-               if (vfio_cfg.vfio_groups[i].group_no == -1) {
+               if (vfio_cfg.vfio_groups[i].group_num == -1) {
                        cur_grp = &vfio_cfg.vfio_groups[i];
                        break;
                }
@@ -318,7 +318,7 @@ vfio_get_group_fd(int iommu_group_no)
        if (internal_config.process_type == RTE_PROC_PRIMARY) {
                /* try regular group format */
                snprintf(filename, sizeof(filename),
-                                VFIO_GROUP_FMT, iommu_group_no);
+                                VFIO_GROUP_FMT, iommu_group_num);
                vfio_group_fd = open(filename, O_RDWR);
                if (vfio_group_fd < 0) {
                        /* if file not found, it's not an error */
@@ -330,7 +330,8 @@ vfio_get_group_fd(int iommu_group_no)
 
                        /* special case: try no-IOMMU path as well */
                        snprintf(filename, sizeof(filename),
-                                       VFIO_NOIOMMU_GROUP_FMT, iommu_group_no);
+                                       VFIO_NOIOMMU_GROUP_FMT,
+                                       iommu_group_num);
                        vfio_group_fd = open(filename, O_RDWR);
                        if (vfio_group_fd < 0) {
                                if (errno != ENOENT) {
@@ -343,7 +344,7 @@ vfio_get_group_fd(int iommu_group_no)
                        /* noiommu group found */
                }
 
-               cur_grp->group_no = iommu_group_no;
+               cur_grp->group_num = iommu_group_num;
                cur_grp->fd = vfio_group_fd;
                vfio_cfg.vfio_active_groups++;
                return vfio_group_fd;
@@ -365,7 +366,7 @@ vfio_get_group_fd(int iommu_group_no)
                        close(socket_fd);
                        return -1;
                }
-               if (vfio_mp_sync_send_request(socket_fd, iommu_group_no) < 0) {
+               if (vfio_mp_sync_send_request(socket_fd, iommu_group_num) < 0) {
                        RTE_LOG(ERR, EAL, "  cannot send group number!\n");
                        close(socket_fd);
                        return -1;
@@ -380,7 +381,7 @@ vfio_get_group_fd(int iommu_group_no)
                        /* if we got the fd, store it and return it */
                        if (vfio_group_fd > 0) {
                                close(socket_fd);
-                               cur_grp->group_no = iommu_group_no;
+                               cur_grp->group_num = iommu_group_num;
                                cur_grp->fd = vfio_group_fd;
                                vfio_cfg.vfio_active_groups++;
                                return vfio_group_fd;
@@ -487,7 +488,7 @@ rte_vfio_clear_group(int vfio_group_fd)
                i = get_vfio_group_idx(vfio_group_fd);
                if (i < 0)
                        return -1;
-               vfio_cfg.vfio_groups[i].group_no = -1;
+               vfio_cfg.vfio_groups[i].group_num = -1;
                vfio_cfg.vfio_groups[i].fd = -1;
                vfio_cfg.vfio_groups[i].devices = 0;
                vfio_cfg.vfio_active_groups--;
@@ -544,11 +545,11 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
                        .argsz = sizeof(group_status)
        };
        int vfio_group_fd;
-       int iommu_group_no;
+       int iommu_group_num;
        int i, ret;
 
        /* get group number */
-       ret = vfio_get_group_no(sysfs_base, dev_addr, &iommu_group_no);
+       ret = rte_vfio_get_group_num(sysfs_base, dev_addr, &iommu_group_num);
        if (ret == 0) {
                RTE_LOG(WARNING, EAL, "  %s not managed by VFIO driver, skipping\n",
                        dev_addr);
@@ -560,7 +561,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
                return -1;
 
        /* get the actual group fd */
-       vfio_group_fd = vfio_get_group_fd(iommu_group_no);
+       vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
        if (vfio_group_fd < 0)
                return -1;
 
@@ -733,7 +734,7 @@ rte_vfio_release_device(const char *sysfs_base, const char *dev_addr,
                        .argsz = sizeof(group_status)
        };
        int vfio_group_fd;
-       int iommu_group_no;
+       int iommu_group_num;
        int ret;
 
        /* we don't want any DMA mapping messages to come while we're detaching
@@ -743,7 +744,7 @@ rte_vfio_release_device(const char *sysfs_base, const char *dev_addr,
        rte_rwlock_read_lock(mem_lock);
 
        /* get group number */
-       ret = vfio_get_group_no(sysfs_base, dev_addr, &iommu_group_no);
+       ret = rte_vfio_get_group_num(sysfs_base, dev_addr, &iommu_group_num);
        if (ret <= 0) {
                RTE_LOG(WARNING, EAL, "  %s not managed by VFIO driver\n",
                        dev_addr);
@@ -753,9 +754,9 @@ rte_vfio_release_device(const char *sysfs_base, const char *dev_addr,
        }
 
        /* get the actual group fd */
-       vfio_group_fd = vfio_get_group_fd(iommu_group_no);
+       vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
        if (vfio_group_fd <= 0) {
-               RTE_LOG(INFO, EAL, "vfio_get_group_fd failed for %s\n",
+               RTE_LOG(INFO, EAL, "rte_vfio_get_group_fd failed for %s\n",
                                   dev_addr);
                ret = -1;
                goto out;
@@ -818,7 +819,7 @@ rte_vfio_enable(const char *modname)
 
        for (i = 0; i < VFIO_MAX_GROUPS; i++) {
                vfio_cfg.vfio_groups[i].fd = -1;
-               vfio_cfg.vfio_groups[i].group_no = -1;
+               vfio_cfg.vfio_groups[i].group_num = -1;
                vfio_cfg.vfio_groups[i].devices = 0;
        }
 
@@ -841,7 +842,7 @@ rte_vfio_enable(const char *modname)
                return 0;
        }
 
-       vfio_cfg.vfio_container_fd = vfio_get_container_fd();
+       vfio_cfg.vfio_container_fd = rte_vfio_get_container_fd();
 
        /* check if we have VFIO driver enabled */
        if (vfio_cfg.vfio_container_fd != -1) {
@@ -919,7 +920,7 @@ vfio_has_supported_extensions(int vfio_container_fd)
 }
 
 int
-vfio_get_container_fd(void)
+rte_vfio_get_container_fd(void)
 {
        int ret, vfio_container_fd;
 
@@ -983,8 +984,8 @@ vfio_get_container_fd(void)
 }
 
 int
-vfio_get_group_no(const char *sysfs_base,
-               const char *dev_addr, int *iommu_group_no)
+rte_vfio_get_group_num(const char *sysfs_base,
+               const char *dev_addr, int *iommu_group_num)
 {
        char linkname[PATH_MAX];
        char filename[PATH_MAX];
@@ -1016,7 +1017,7 @@ vfio_get_group_no(const char *sysfs_base,
        errno = 0;
        group_tok = tok[ret - 1];
        end = group_tok;
-       *iommu_group_no = strtol(group_tok, &end, 10);
+       *iommu_group_num = strtol(group_tok, &end, 10);
        if ((end != group_tok && *end != '\0') || errno != 0) {
                RTE_LOG(ERR, EAL, "  %s error parsing IOMMU number!\n", dev_addr);
                return -1;

diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.h b/lib/librte_eal/linuxapp/eal/eal_vfio.h
index 549f442..c788bba 100644 (file)
--- a/lib/librte_eal/linuxapp/eal/eal_vfio.h
+++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h
@@ -81,12 +81,6 @@ struct vfio_iommu_spapr_tce_info {
 #define RTE_VFIO_SPAPR VFIO_SPAPR_TCE_v2_IOMMU
 #endif
 
-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
-#define RTE_VFIO_NOIOMMU 8
-#else
-#define RTE_VFIO_NOIOMMU VFIO_NOIOMMU_IOMMU
-#endif
-
 #define VFIO_MAX_GROUPS RTE_MAX_VFIO_GROUPS
 
 /*
@@ -103,7 +97,7 @@ int vfio_mp_sync_connect_to_primary(void);
  * the group fd via an ioctl() call.
  */
 struct vfio_group {
-       int group_no;
+       int group_num;
        int fd;
        int devices;
 };
@@ -145,21 +139,6 @@ vfio_set_iommu_type(int vfio_container_fd);
 int
 vfio_has_supported_extensions(int vfio_container_fd);
 
-/* open container fd or get an existing one */
-int
-vfio_get_container_fd(void);
-
-/* parse IOMMU group number for a device
- * returns 1 on success, -1 for errors, 0 for non-existent group
- */
-int
-vfio_get_group_no(const char *sysfs_base,
-               const char *dev_addr, int *iommu_group_no);
-
-/* open group fd or get an existing one */
-int
-vfio_get_group_fd(int iommu_group_no);
-
 int vfio_mp_sync_setup(void);
 
 #define SOCKET_REQ_CONTAINER 0x100

diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c b/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c
index 7cc3c15..e19b571 100644 (file)
--- a/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c
+++ b/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c
@@ -267,7 +267,7 @@ vfio_mp_sync_thread(void __rte_unused * arg)
 
                switch (ret) {
                case SOCKET_REQ_CONTAINER:
-                       fd = vfio_get_container_fd();
+                       fd = rte_vfio_get_container_fd();
                        if (fd < 0)
                                vfio_mp_sync_send_request(conn_sock, SOCKET_ERR);
                        else
@@ -283,7 +283,7 @@ vfio_mp_sync_thread(void __rte_unused * arg)
                                continue;
                        }
 
-                       fd = vfio_get_group_fd(vfio_data);
+                       fd = rte_vfio_get_group_fd(vfio_data);
 
                        if (fd < 0)
                                vfio_mp_sync_send_request(conn_sock, SOCKET_ERR);

diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 2b5b1dc..603c744 100644 (file)
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -286,5 +286,8 @@ EXPERIMENTAL {
        rte_socket_id_by_idx;
        rte_vfio_dma_map;
        rte_vfio_dma_unmap;
+       rte_vfio_get_container_fd;
+       rte_vfio_get_group_fd;
+       rte_vfio_get_group_num;
 
 } DPDK_18.02;