From: Santosh Shukla Date: Fri, 6 Oct 2017 11:03:40 +0000 (+0530) Subject: pci: get IOMMU class on Linux X-Git-Tag: spdx-start~1740 X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=815c7deaed2d9e325968e82cb599984088a5c55a;p=dpdk.git pci: get IOMMU class on Linux Get iommu class of PCI device on the bus and returns preferred iova mapping mode for that bus. Patch also introduces RTE_PCI_DRV_IOVA_AS_VA drv flag. Flag used when driver needs to operate in iova=va mode. Algorithm for iova scheme selection for PCI bus: 0. If no device bound then return with RTE_IOVA_DC mapping mode, else goto 1). 1. Look for device attached to vfio kdrv and has .drv_flag set to RTE_PCI_DRV_IOVA_AS_VA. 2. Look for any device attached to UIO class of driver. 3. Check for vfio-noiommu mode enabled. If 2) & 3) is false and 1) is true then select mapping scheme as RTE_IOVA_VA. Otherwise use default mapping scheme (RTE_IOVA_PA). Signed-off-by: Santosh Shukla Signed-off-by: Jerin Jacob Reviewed-by: Maxime Coquelin Reviewed-by: Anatoly Burakov Acked-by: Hemant Agrawal Tested-by: Hemant Agrawal --- diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h index 0e36de0938..7a61898e5d 100644 --- a/lib/librte_eal/common/include/rte_pci.h +++ b/lib/librte_eal/common/include/rte_pci.h @@ -202,6 +202,8 @@ struct rte_pci_bus { #define RTE_PCI_DRV_INTR_RMV 0x0010 /** Device driver needs to keep mapped resources if unsupported dev detected */ #define RTE_PCI_DRV_KEEP_MAPPED_RES 0x0020 +/** Device driver supports IOVA as VA */ +#define RTE_PCI_DRV_IOVA_AS_VA 0X0040 /** * A structure describing a PCI mapping. diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c index b1d6fbdce0..b4dbf953af 100644 --- a/lib/librte_eal/linuxapp/eal/eal_pci.c +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c @@ -45,6 +45,7 @@ #include "eal_filesystem.h" #include "eal_private.h" #include "eal_pci_init.h" +#include "eal_vfio.h" /** * @file @@ -487,12 +488,98 @@ error: return -1; } +/* + * Is pci device bound to any kdrv + */ +static inline int +pci_one_device_is_bound(void) +{ + struct rte_pci_device *dev = NULL; + int ret = 0; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_UNKNOWN || + dev->kdrv == RTE_KDRV_NONE) { + continue; + } else { + ret = 1; + break; + } + } + return ret; +} + +/* + * Any one of the device bound to uio + */ +static inline int +pci_one_device_bound_uio(void) +{ + struct rte_pci_device *dev = NULL; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_IGB_UIO || + dev->kdrv == RTE_KDRV_UIO_GENERIC) { + return 1; + } + } + return 0; +} + +/* + * Any one of the device has iova as va + */ +static inline int +pci_one_device_has_iova_va(void) +{ + struct rte_pci_device *dev = NULL; + struct rte_pci_driver *drv = NULL; + + FOREACH_DRIVER_ON_PCIBUS(drv) { + if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_VFIO && + rte_pci_match(drv, dev)) + return 1; + } + } + } + return 0; +} + /* * Get iommu class of PCI devices on the bus. */ enum rte_iova_mode rte_pci_get_iommu_class(void) { + bool is_bound; + bool is_vfio_noiommu_enabled = true; + bool has_iova_va; + bool is_bound_uio; + + is_bound = pci_one_device_is_bound(); + if (!is_bound) + return RTE_IOVA_DC; + + has_iova_va = pci_one_device_has_iova_va(); + is_bound_uio = pci_one_device_bound_uio(); +#ifdef VFIO_PRESENT + is_vfio_noiommu_enabled = vfio_noiommu_is_enabled() == true ? + true : false; +#endif + + if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled) + return RTE_IOVA_VA; + + if (has_iova_va) { + RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa will be used because.. "); + if (is_vfio_noiommu_enabled) + RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n"); + if (is_bound_uio) + RTE_LOG(WARNING, EAL, "few device bound to UIO\n"); + } + return RTE_IOVA_PA; } diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c b/lib/librte_eal/linuxapp/eal/eal_vfio.c index 946df7e31f..c8a97b7e75 100644 --- a/lib/librte_eal/linuxapp/eal/eal_vfio.c +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c @@ -816,4 +816,23 @@ vfio_noiommu_dma_map(int __rte_unused vfio_container_fd) return 0; } +int +vfio_noiommu_is_enabled(void) +{ + int fd, ret, cnt __rte_unused; + char c; + + ret = -1; + fd = open(VFIO_NOIOMMU_MODE, O_RDONLY); + if (fd < 0) + return -1; + + cnt = read(fd, &c, 1); + if (c == 'Y') + ret = 1; + + close(fd); + return ret; +} + #endif diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.h b/lib/librte_eal/linuxapp/eal/eal_vfio.h index 5ff63e5d75..26ea8e1197 100644 --- a/lib/librte_eal/linuxapp/eal/eal_vfio.h +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h @@ -150,6 +150,8 @@ struct vfio_config { #define VFIO_NOIOMMU_GROUP_FMT "/dev/vfio/noiommu-%u" #define VFIO_GET_REGION_ADDR(x) ((uint64_t) x << 40ULL) #define VFIO_GET_REGION_IDX(x) (x >> 40) +#define VFIO_NOIOMMU_MODE \ + "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode" /* DMA mapping function prototype. * Takes VFIO container fd as a parameter. @@ -210,6 +212,8 @@ int pci_vfio_is_enabled(void); int vfio_mp_sync_setup(void); +int vfio_noiommu_is_enabled(void); + #define SOCKET_REQ_CONTAINER 0x100 #define SOCKET_REQ_GROUP 0x200 #define SOCKET_CLR_GROUP 0x300