From a0dede62a5372da0a9b67859962e64108a10b916 Mon Sep 17 00:00:00 2001 From: Vamsi Attunuru Date: Sun, 17 Nov 2019 16:12:44 +0100 Subject: [PATCH] eal/linux: remove KNI restriction on IOVA Now that KNI supports VA (with kernel versions starting 4.6.0), we can accept IOVA as VA, but KNI must be configured for this. Pass iova_mode when creating KNI netdevs. So far, IOVA detection policy forced IOVA as PA when KNI is loaded, whatever the buses IOVA requirements were. We can now use IOVA as VA, but this comes with a cost in KNI. When no constraint is expressed by the buses, keep the current behavior of choosing PA. Note: this change supposes that dpdk is built on the same kernel than the target system kernel; no objection has been expressed on this topic. Signed-off-by: Vamsi Attunuru Signed-off-by: Kiran Kumar K Signed-off-by: David Marchand Reviewed-by: Jerin Jacob --- doc/guides/prog_guide/env_abstraction_layer.rst | 3 +++ doc/guides/prog_guide/kernel_nic_interface.rst | 14 ++++++++++++++ doc/guides/rel_notes/release_19_11.rst | 11 +++++++++++ lib/librte_eal/linux/eal/eal.c | 11 +++++++++-- lib/librte_kni/rte_kni.c | 5 +++++ 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/doc/guides/prog_guide/env_abstraction_layer.rst b/doc/guides/prog_guide/env_abstraction_layer.rst index cd8e3003e4..6e7c2080a4 100644 --- a/doc/guides/prog_guide/env_abstraction_layer.rst +++ b/doc/guides/prog_guide/env_abstraction_layer.rst @@ -475,6 +475,9 @@ devices would fail anyway. ``RTE_PCI_DRV_NEED_IOVA_AS_VA`` flag is used to dictate that this PCI driver can only work in RTE_IOVA_VA mode. + When the KNI kernel module is detected, RTE_IOVA_PA mode is preferred as a + performance penalty is expected in RTE_IOVA_VA mode. + IOVA Mode Configuration ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/guides/prog_guide/kernel_nic_interface.rst b/doc/guides/prog_guide/kernel_nic_interface.rst index e12634ddc8..c4479ffbf7 100644 --- a/doc/guides/prog_guide/kernel_nic_interface.rst +++ b/doc/guides/prog_guide/kernel_nic_interface.rst @@ -300,6 +300,20 @@ The sk_buff is then freed and the mbuf sent in the tx_q FIFO. The DPDK TX thread dequeues the mbuf and sends it to the PMD via ``rte_eth_tx_burst()``. It then puts the mbuf back in the cache. +IOVA = VA: Support +------------------ + +KNI operates in IOVA_VA scheme when + +- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) and +- EAL option `iova-mode=va` is passed or bus IOVA scheme in the DPDK is selected + as RTE_IOVA_VA. + +Due to IOVA to KVA address translations, based on the KNI use case there +can be a performance impact. For mitigation, forcing IOVA to PA via EAL +"--iova-mode=pa" option can be used, IOVA_DC bus iommu scheme can also +result in IOVA as PA. + Ethtool ------- diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst index c0045a91ff..21be600abd 100644 --- a/doc/guides/rel_notes/release_19_11.rst +++ b/doc/guides/rel_notes/release_19_11.rst @@ -294,6 +294,17 @@ New Features See :doc:`../prog_guide/lto` for more information: +* **Added IOVA as VA support for KNI.** + + * Added IOVA = VA support for KNI, KNI can operate in IOVA = VA mode when + `iova-mode=va` EAL option is passed to the application or when bus IOVA + scheme is selected as RTE_IOVA_VA. This mode only works on Linux Kernel + versions 4.6.0 and above. + + * Due to IOVA to KVA address translations, based on the KNI use case there + can be a performance impact. For mitigation, forcing IOVA to PA via EAL + "--iova-mode=pa" option can be used, IOVA_DC bus iommu scheme can also + result in IOVA as PA. Removed Items diff --git a/lib/librte_eal/linux/eal/eal.c b/lib/librte_eal/linux/eal/eal.c index 9e2d50cfba..b5b71500c3 100644 --- a/lib/librte_eal/linux/eal/eal.c +++ b/lib/librte_eal/linux/eal/eal.c @@ -1073,6 +1073,11 @@ rte_eal_init(int argc, char **argv) */ iova_mode = RTE_IOVA_VA; RTE_LOG(DEBUG, EAL, "Physical addresses are unavailable, selecting IOVA as VA mode.\n"); +#if defined(RTE_LIBRTE_KNI) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) + } else if (rte_eal_check_module("rte_kni") == 1) { + iova_mode = RTE_IOVA_PA; + RTE_LOG(DEBUG, EAL, "KNI is loaded, selecting IOVA as PA mode for better KNI perfomance.\n"); +#endif } else if (is_iommu_enabled()) { /* we have an IOMMU, pick IOVA as VA mode */ iova_mode = RTE_IOVA_VA; @@ -1085,8 +1090,10 @@ rte_eal_init(int argc, char **argv) RTE_LOG(DEBUG, EAL, "IOMMU is not available, selecting IOVA as PA mode.\n"); } } -#ifdef RTE_LIBRTE_KNI - /* Workaround for KNI which requires physical address to work */ +#if defined(RTE_LIBRTE_KNI) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + /* Workaround for KNI which requires physical address to work + * in kernels < 4.6 + */ if (iova_mode == RTE_IOVA_VA && rte_eal_check_module("rte_kni") == 1) { if (phys_addrs) { diff --git a/lib/librte_kni/rte_kni.c b/lib/librte_kni/rte_kni.c index 7fbcf22010..86995fc817 100644 --- a/lib/librte_kni/rte_kni.c +++ b/lib/librte_kni/rte_kni.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -97,10 +98,12 @@ static volatile int kni_fd = -1; int rte_kni_init(unsigned int max_kni_ifaces __rte_unused) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) if (rte_eal_iova_mode() != RTE_IOVA_PA) { RTE_LOG(ERR, KNI, "KNI requires IOVA as PA\n"); return -1; } +#endif /* Check FD and open */ if (kni_fd < 0) { @@ -302,6 +305,8 @@ rte_kni_alloc(struct rte_mempool *pktmbuf_pool, kni->group_id = conf->group_id; kni->mbuf_size = conf->mbuf_size; + dev_info.iova_mode = (rte_eal_iova_mode() == RTE_IOVA_VA) ? 1 : 0; + ret = ioctl(kni_fd, RTE_KNI_IOCTL_CREATE, &dev_info); if (ret < 0) goto ioctl_fail; -- 2.20.1