From 26b683b4f7d042a136b60d1887cfba80dceb9b62 Mon Sep 17 00:00:00 2001 From: Jianfeng Tan Date: Tue, 17 Jan 2017 08:00:03 +0000 Subject: [PATCH] net/virtio: setup Rx queue interrupts This patch mainly allocates structure to store queue/irq mapping, and configure queue/irq mapping down through PCI ops. It also creates eventfds for each Rx queue and tell the kernel about the eventfd/intr binding. Note: So far, we hard-code 1:1 queue/irq mapping (each rx queue has one exclusive interrupt), like this: vec 0 -> config irq vec 1 -> rxq0 vec 2 -> rxq1 ... which means, the "vectors" option of QEMU should be configured with a value >= N+1 (N is the number of the queue pairs). Signed-off-by: Jianfeng Tan Tested-by: Lei Yao Acked-by: Yuanhan Liu --- doc/guides/nics/features/virtio.ini | 1 + doc/guides/nics/features/virtio_vec.ini | 1 + doc/guides/nics/virtio.rst | 60 ++++++++++++++++++ doc/guides/rel_notes/release_17_02.rst | 13 ++++ drivers/net/virtio/virtio_ethdev.c | 84 +++++++++++++++++++++++++ 5 files changed, 159 insertions(+) diff --git a/doc/guides/nics/features/virtio.ini b/doc/guides/nics/features/virtio.ini index 1d996c657b..5164937aed 100644 --- a/doc/guides/nics/features/virtio.ini +++ b/doc/guides/nics/features/virtio.ini @@ -5,6 +5,7 @@ ; [Features] Link status = Y +Rx interrupt = Y Queue start/stop = Y Scattered Rx = Y Promiscuous mode = Y diff --git a/doc/guides/nics/features/virtio_vec.ini b/doc/guides/nics/features/virtio_vec.ini index 6dc7cf0243..ec93f5c405 100644 --- a/doc/guides/nics/features/virtio_vec.ini +++ b/doc/guides/nics/features/virtio_vec.ini @@ -5,6 +5,7 @@ ; [Features] Link status = Y +Rx interrupt = Y Queue start/stop = Y Promiscuous mode = Y Allmulticast mode = Y diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst index c90e5175c6..dcf4d35445 100644 --- a/doc/guides/nics/virtio.rst +++ b/doc/guides/nics/virtio.rst @@ -87,6 +87,8 @@ In this release, the virtio PMD driver provides the basic functionality of packe * Virtio supports Link State interrupt. +* Virtio supports Rx interrupt (so far, only support 1:1 mapping for queue/interrupt). + * Virtio supports software vlan stripping and inserting. * Virtio supports using port IO to get PCI resource when uio/igb_uio module is not available. @@ -274,3 +276,61 @@ Example of using the vector version of the virtio poll mode driver in ``testpmd``:: testpmd -c 0x7 -n 4 -- -i --txqflags=0xF01 --rxq=1 --txq=1 --nb-cores=1 + + +Interrupt mode +-------------- + +.. _virtio_interrupt_mode: + +There are three kinds of interrupts from a virtio device over PCI bus: config +interrupt, Rx interrupts, and Tx interrupts. Config interrupt is used for +notification of device configuration changes, especially link status (lsc). +Interrupt mode is translated into Rx interrupts in the context of DPDK. + +Prerequisites for Rx interrupts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To support Rx interrupts, +#. Check if guest kernel supports VFIO-NOIOMMU: + + Linux started to support VFIO-NOIOMMU since 4.8.0. Make sure the guest + kernel is compiled with: + + .. code-block:: console + + CONFIG_VFIO_NOIOMMU=y + +#. Properly set msix vectors when starting VM: + + Enable multi-queue when starting VM, and specify msix vectors in qemu + cmdline. (N+1) is the minimum, and (2N+2) is mostly recommended. + + .. code-block:: console + + $(QEMU) ... -device virtio-net-pci,mq=on,vectors=2N+2 ... + +#. In VM, insert vfio module in NOIOMMU mode: + + .. code-block:: console + + modprobe vfio enable_unsafe_noiommu_mode=1 + modprobe vfio-pci + +#. In VM, bind the virtio device with vfio-pci: + + .. code-block:: console + + python tools/dpdk-devbind.py -b vfio-pci 00:03.0 + +Example +~~~~~~~ + +Here we use l3fwd-power as an example to show how to get started. + + Example: + + .. code-block:: console + + $ l3fwd-power -c 0x3 -- -p 1 -P --config="(0,0,1)" \ + --no-numa --parse-ptype diff --git a/doc/guides/rel_notes/release_17_02.rst b/doc/guides/rel_notes/release_17_02.rst index a230724bbf..73849b5459 100644 --- a/doc/guides/rel_notes/release_17_02.rst +++ b/doc/guides/rel_notes/release_17_02.rst @@ -78,6 +78,19 @@ New Features it might enter into kernel space to wake up those kthreads if necessary). +* **Added virtio Rx interrupt suppprt.** + + This feature enables Rx interrupt mode for virtio pci net devices as + binded to VFIO (noiommu mode) and drived by virtio PMD. + + With this feature, virtio PMD can switch between polling mode and + interrupt mode, to achieve best performance, and at the same time save + power. It can work on both legacy and modern virtio devices. At this mode, + each rxq is mapped with an exluded MSIx interrupt. + + See the :ref:`Virtio Interrupt Mode ` documentation + for more information. + Resolved Issues --------------- diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index f356cf12b5..80253820ff 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -1210,6 +1210,82 @@ rx_func_get(struct rte_eth_dev *eth_dev) eth_dev->rx_pkt_burst = &virtio_recv_pkts; } +/* Only support 1:1 queue/interrupt mapping so far. + * TODO: support n:1 queue/interrupt mapping when there are limited number of + * interrupt vectors (data->dev_private; + + PMD_INIT_LOG(INFO, "queue/interrupt binding\n"); + for (i = 0; i < dev->data->nb_rx_queues; ++i) { + dev->intr_handle->intr_vec[i] = i + 1; + if (VTPCI_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) == + VIRTIO_MSI_NO_VECTOR) { + PMD_DRV_LOG(ERR, "failed to set queue vector"); + return -EBUSY; + } + } + + return 0; +} + +static int +virtio_configure_intr(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw = dev->data->dev_private; + + if (!rte_intr_cap_multiple(dev->intr_handle)) { + PMD_INIT_LOG(ERR, "Multiple intr vector not supported"); + return -ENOTSUP; + } + + if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) { + PMD_INIT_LOG(ERR, "Fail to create eventfd"); + return -1; + } + + if (!dev->intr_handle->intr_vec) { + dev->intr_handle->intr_vec = + rte_zmalloc("intr_vec", + hw->max_queue_pairs * sizeof(int), 0); + if (!dev->intr_handle->intr_vec) { + PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors", + hw->max_queue_pairs); + return -ENOMEM; + } + } + + /* Re-register callback to update max_intr */ + rte_intr_callback_unregister(dev->intr_handle, + virtio_interrupt_handler, + dev); + rte_intr_callback_register(dev->intr_handle, + virtio_interrupt_handler, + dev); + + /* DO NOT try to remove this! This function will enable msix, or QEMU + * will encounter SIGSEGV when DRIVER_OK is sent. + * And for legacy devices, this should be done before queue/vec binding + * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR + * (22) will be ignored. + */ + if (rte_intr_enable(dev->intr_handle) < 0) { + PMD_DRV_LOG(ERR, "interrupt enable failed"); + return -1; + } + + if (virtio_queues_bind_intr(dev) < 0) { + PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt"); + return -1; + } + + return 0; +} + /* reset device and renegotiate features if needed */ static int virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) @@ -1306,6 +1382,14 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) ret = virtio_alloc_queues(eth_dev); if (ret < 0) return ret; + + if (eth_dev->data->dev_conf.intr_conf.rxq) { + if (virtio_configure_intr(eth_dev) < 0) { + PMD_INIT_LOG(ERR, "failed to configure interrupt"); + return -1; + } + } + vtpci_reinit_complete(hw); if (pci_dev) -- 2.20.1