X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_user_ethdev.c;h=6003f6d50f6360a711054f7d2524651af7fff70e;hb=d0131e49c7fca45f4111eedab71f6dbd73bacd61;hp=6afbf39d2cd0c3d731a9cb56d75e4dad0f50bc03;hpb=2695c6df698400c0d9a48b982f1e902a43018305;p=dpdk.git diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c index 6afbf39d2c..6003f6d50f 100644 --- a/drivers/net/virtio/virtio_user_ethdev.c +++ b/drivers/net/virtio/virtio_user_ethdev.c @@ -1,43 +1,19 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation */ #include #include #include +#include +#include #include #include -#include +#include +#include +#include +#include #include "virtio_ethdev.h" #include "virtio_logs.h" @@ -45,10 +21,161 @@ #include "virtqueue.h" #include "virtio_rxtx.h" #include "virtio_user/virtio_user_dev.h" +#include "virtio_user/vhost.h" #define virtio_user_get_dev(hw) \ ((struct virtio_user_dev *)(hw)->virtio_user_dev) +static void +virtio_user_reset_queues_packed(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw = dev->data->dev_private; + struct virtnet_rx *rxvq; + struct virtnet_tx *txvq; + uint16_t i; + + /* Add lock to avoid queue contention. */ + rte_spinlock_lock(&hw->state_lock); + hw->started = 0; + + /* + * Waitting for datapath to complete before resetting queues. + * 1 ms should be enough for the ongoing Tx/Rx function to finish. + */ + rte_delay_ms(1); + + /* Vring reset for each Tx queue and Rx queue. */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxvq = dev->data->rx_queues[i]; + virtqueue_rxvq_reset_packed(rxvq->vq); + virtio_dev_rx_queue_setup_finish(dev, i); + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txvq = dev->data->tx_queues[i]; + virtqueue_txvq_reset_packed(txvq->vq); + } + + hw->started = 1; + rte_spinlock_unlock(&hw->state_lock); +} + + +static int +virtio_user_server_reconnect(struct virtio_user_dev *dev) +{ + int ret; + int connectfd; + struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id]; + struct virtio_hw *hw = eth_dev->data->dev_private; + uint64_t protocol_features; + + connectfd = accept(dev->listenfd, NULL, NULL); + if (connectfd < 0) + return -1; + + dev->vhostfd = connectfd; + if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES, + &dev->device_features) < 0) { + PMD_INIT_LOG(ERR, "get_features failed: %s", + strerror(errno)); + return -1; + } + + if (dev->device_features & + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) { + if (dev->ops->send_request(dev, + VHOST_USER_GET_PROTOCOL_FEATURES, + &protocol_features)) + return -1; + + dev->protocol_features &= protocol_features; + + if (dev->ops->send_request(dev, + VHOST_USER_SET_PROTOCOL_FEATURES, + &dev->protocol_features)) + return -1; + + if (!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_MQ))) + dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); + } + + dev->device_features |= dev->frontend_features; + + /* umask vhost-user unsupported features */ + dev->device_features &= ~(dev->unsupported_features); + + dev->features &= dev->device_features; + + /* For packed ring, resetting queues is required in reconnection. */ + if (vtpci_packed_queue(hw) && + (vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { + PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" + " when packed ring reconnecting."); + virtio_user_reset_queues_packed(eth_dev); + } + + ret = virtio_user_start_device(dev); + if (ret < 0) + return -1; + + if (dev->queue_pairs > 1) { + ret = virtio_user_handle_mq(dev, dev->queue_pairs); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); + return -1; + } + } + if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { + if (rte_intr_disable(eth_dev->intr_handle) < 0) { + PMD_DRV_LOG(ERR, "interrupt disable failed"); + return -1; + } + rte_intr_callback_unregister(eth_dev->intr_handle, + virtio_interrupt_handler, + eth_dev); + eth_dev->intr_handle->fd = connectfd; + rte_intr_callback_register(eth_dev->intr_handle, + virtio_interrupt_handler, eth_dev); + + if (rte_intr_enable(eth_dev->intr_handle) < 0) { + PMD_DRV_LOG(ERR, "interrupt enable failed"); + return -1; + } + } + PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); + return 0; +} + +static void +virtio_user_delayed_handler(void *param) +{ + struct virtio_hw *hw = (struct virtio_hw *)param; + struct rte_eth_dev *eth_dev = &rte_eth_devices[hw->port_id]; + struct virtio_user_dev *dev = virtio_user_get_dev(hw); + + if (rte_intr_disable(eth_dev->intr_handle) < 0) { + PMD_DRV_LOG(ERR, "interrupt disable failed"); + return; + } + rte_intr_callback_unregister(eth_dev->intr_handle, + virtio_interrupt_handler, eth_dev); + if (dev->is_server) { + if (dev->vhostfd >= 0) { + close(dev->vhostfd); + dev->vhostfd = -1; + } + eth_dev->intr_handle->fd = dev->listenfd; + rte_intr_callback_register(eth_dev->intr_handle, + virtio_interrupt_handler, eth_dev); + if (rte_intr_enable(eth_dev->intr_handle) < 0) { + PMD_DRV_LOG(ERR, "interrupt enable failed"); + return; + } + } +} + static void virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, void *dst, int length) @@ -57,14 +184,54 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, struct virtio_user_dev *dev = virtio_user_get_dev(hw); if (offset == offsetof(struct virtio_net_config, mac) && - length == ETHER_ADDR_LEN) { - for (i = 0; i < ETHER_ADDR_LEN; ++i) + length == RTE_ETHER_ADDR_LEN) { + for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) ((uint8_t *)dst)[i] = dev->mac_addr[i]; return; } - if (offset == offsetof(struct virtio_net_config, status)) - *(uint16_t *)dst = dev->status; + if (offset == offsetof(struct virtio_net_config, status)) { + char buf[128]; + + if (dev->vhostfd >= 0) { + int r; + int flags; + + flags = fcntl(dev->vhostfd, F_GETFL); + if (fcntl(dev->vhostfd, F_SETFL, + flags | O_NONBLOCK) == -1) { + PMD_DRV_LOG(ERR, "error setting O_NONBLOCK flag"); + return; + } + r = recv(dev->vhostfd, buf, 128, MSG_PEEK); + if (r == 0 || (r < 0 && errno != EAGAIN)) { + dev->net_status &= (~VIRTIO_NET_S_LINK_UP); + PMD_DRV_LOG(ERR, "virtio-user port %u is down", + hw->port_id); + + /* This function could be called in the process + * of interrupt handling, callback cannot be + * unregistered here, set an alarm to do it. + */ + rte_eal_alarm_set(1, + virtio_user_delayed_handler, + (void *)hw); + } else { + dev->net_status |= VIRTIO_NET_S_LINK_UP; + } + if (fcntl(dev->vhostfd, F_SETFL, + flags & ~O_NONBLOCK) == -1) { + PMD_DRV_LOG(ERR, "error clearing O_NONBLOCK flag"); + return; + } + } else if (dev->is_server) { + dev->net_status &= (~VIRTIO_NET_S_LINK_UP); + if (virtio_user_server_reconnect(dev) >= 0) + dev->net_status |= VIRTIO_NET_S_LINK_UP; + } + + *(uint16_t *)dst = dev->net_status; + } if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs)) *(uint16_t *)dst = dev->max_queue_pairs; @@ -78,30 +245,33 @@ virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset, struct virtio_user_dev *dev = virtio_user_get_dev(hw); if ((offset == offsetof(struct virtio_net_config, mac)) && - (length == ETHER_ADDR_LEN)) - for (i = 0; i < ETHER_ADDR_LEN; ++i) + (length == RTE_ETHER_ADDR_LEN)) + for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) dev->mac_addr[i] = ((const uint8_t *)src)[i]; else - PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d\n", + PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d", offset, length); } static void -virtio_user_set_status(struct virtio_hw *hw, uint8_t status) +virtio_user_reset(struct virtio_hw *hw) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); - if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) - virtio_user_start_device(dev); - dev->status = status; + if (dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK) + virtio_user_stop_device(dev); } static void -virtio_user_reset(struct virtio_hw *hw) +virtio_user_set_status(struct virtio_hw *hw, uint8_t status) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); - virtio_user_stop_device(dev); + if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) + virtio_user_start_device(dev); + else if (status == VIRTIO_CONFIG_STATUS_RESET) + virtio_user_reset(hw); + dev->status = status; } static uint8_t @@ -117,7 +287,8 @@ virtio_user_get_features(struct virtio_hw *hw) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); - return dev->features; + /* unmask feature bits defined in vhost user protocol */ + return dev->device_features & VIRTIO_PMD_SUPPORTED_GUEST_FEATURES; } static void @@ -125,23 +296,32 @@ virtio_user_set_features(struct virtio_hw *hw, uint64_t features) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); - dev->features = features; + dev->features = features & dev->device_features; } static uint8_t virtio_user_get_isr(struct virtio_hw *hw __rte_unused) { - /* When config interrupt happens, driver calls this function to query - * what kinds of change happen. Interrupt mode not supported for now. + /* rxq interrupts and config interrupt are separated in virtio-user, + * here we only report config change. */ - return 0; + return VIRTIO_PCI_ISR_CONFIG; } static uint16_t virtio_user_set_config_irq(struct virtio_hw *hw __rte_unused, uint16_t vec __rte_unused) { - return VIRTIO_MSI_NO_VECTOR; + return 0; +} + +static uint16_t +virtio_user_set_queue_irq(struct virtio_hw *hw __rte_unused, + struct virtqueue *vq __rte_unused, + uint16_t vec) +{ + /* pretend we have done that */ + return vec; } /* This function is to get the queue size, aka, number of descs, of a specified @@ -157,10 +337,38 @@ virtio_user_get_queue_num(struct virtio_hw *hw, uint16_t queue_id __rte_unused) return dev->queue_size; } -static int -virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) +static void +virtio_user_setup_queue_packed(struct virtqueue *vq, + struct virtio_user_dev *dev) +{ + uint16_t queue_idx = vq->vq_queue_index; + struct vring_packed *vring; + uint64_t desc_addr; + uint64_t avail_addr; + uint64_t used_addr; + uint16_t i; + + vring = &dev->packed_vrings[queue_idx]; + desc_addr = (uintptr_t)vq->vq_ring_virt_mem; + avail_addr = desc_addr + vq->vq_nentries * + sizeof(struct vring_packed_desc); + used_addr = RTE_ALIGN_CEIL(avail_addr + + sizeof(struct vring_packed_desc_event), + VIRTIO_PCI_VRING_ALIGN); + vring->num = vq->vq_nentries; + vring->desc = (void *)(uintptr_t)desc_addr; + vring->driver = (void *)(uintptr_t)avail_addr; + vring->device = (void *)(uintptr_t)used_addr; + dev->packed_queues[queue_idx].avail_wrap_counter = true; + dev->packed_queues[queue_idx].used_wrap_counter = true; + + for (i = 0; i < vring->num; i++) + vring->desc[i].flags = 0; +} + +static void +virtio_user_setup_queue_split(struct virtqueue *vq, struct virtio_user_dev *dev) { - struct virtio_user_dev *dev = virtio_user_get_dev(hw); uint16_t queue_idx = vq->vq_queue_index; uint64_t desc_addr, avail_addr, used_addr; @@ -174,6 +382,17 @@ virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) dev->vrings[queue_idx].desc = (void *)(uintptr_t)desc_addr; dev->vrings[queue_idx].avail = (void *)(uintptr_t)avail_addr; dev->vrings[queue_idx].used = (void *)(uintptr_t)used_addr; +} + +static int +virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) +{ + struct virtio_user_dev *dev = virtio_user_get_dev(hw); + + if (vtpci_packed_queue(hw)) + virtio_user_setup_queue_packed(vq, dev); + else + virtio_user_setup_queue_split(vq, dev); return 0; } @@ -203,25 +422,28 @@ virtio_user_notify_queue(struct virtio_hw *hw, struct virtqueue *vq) struct virtio_user_dev *dev = virtio_user_get_dev(hw); if (hw->cvq && (hw->cvq->vq == vq)) { - virtio_user_handle_cq(dev, vq->vq_queue_index); + if (vtpci_packed_queue(vq->hw)) + virtio_user_handle_cq_packed(dev, vq->vq_queue_index); + else + virtio_user_handle_cq(dev, vq->vq_queue_index); return; } if (write(dev->kickfds[vq->vq_queue_index], &buf, sizeof(buf)) < 0) - PMD_DRV_LOG(ERR, "failed to kick backend: %s\n", + PMD_DRV_LOG(ERR, "failed to kick backend: %s", strerror(errno)); } -static const struct virtio_pci_ops virtio_user_ops = { +const struct virtio_pci_ops virtio_user_ops = { .read_dev_cfg = virtio_user_read_dev_config, .write_dev_cfg = virtio_user_write_dev_config, - .reset = virtio_user_reset, .get_status = virtio_user_get_status, .set_status = virtio_user_set_status, .get_features = virtio_user_get_features, .set_features = virtio_user_set_features, .get_isr = virtio_user_get_isr, .set_config_irq = virtio_user_set_config_irq, + .set_queue_irq = virtio_user_set_queue_irq, .get_queue_num = virtio_user_get_queue_num, .setup_queue = virtio_user_setup_queue, .del_queue = virtio_user_del_queue, @@ -239,12 +461,27 @@ static const char *valid_args[] = { VIRTIO_USER_ARG_PATH, #define VIRTIO_USER_ARG_QUEUE_SIZE "queue_size" VIRTIO_USER_ARG_QUEUE_SIZE, +#define VIRTIO_USER_ARG_INTERFACE_NAME "iface" + VIRTIO_USER_ARG_INTERFACE_NAME, +#define VIRTIO_USER_ARG_SERVER_MODE "server" + VIRTIO_USER_ARG_SERVER_MODE, +#define VIRTIO_USER_ARG_MRG_RXBUF "mrg_rxbuf" + VIRTIO_USER_ARG_MRG_RXBUF, +#define VIRTIO_USER_ARG_IN_ORDER "in_order" + VIRTIO_USER_ARG_IN_ORDER, +#define VIRTIO_USER_ARG_PACKED_VQ "packed_vq" + VIRTIO_USER_ARG_PACKED_VQ, +#define VIRTIO_USER_ARG_SPEED "speed" + VIRTIO_USER_ARG_SPEED, +#define VIRTIO_USER_ARG_VECTORIZED "vectorized" + VIRTIO_USER_ARG_VECTORIZED, NULL }; #define VIRTIO_USER_DEF_CQ_EN 0 #define VIRTIO_USER_DEF_Q_NUM 1 #define VIRTIO_USER_DEF_Q_SZ 256 +#define VIRTIO_USER_DEF_SERVER_MODE 0 static int get_string_arg(const char *key __rte_unused, @@ -255,6 +492,9 @@ get_string_arg(const char *key __rte_unused, *(char **)extra_args = strdup(value); + if (!*(char **)extra_args) + return -ENOMEM; + return 0; } @@ -262,64 +502,76 @@ static int get_integer_arg(const char *key __rte_unused, const char *value, void *extra_args) { + uint64_t integer = 0; if (!value || !extra_args) return -EINVAL; - - *(uint64_t *)extra_args = strtoull(value, NULL, 0); - - return 0; + errno = 0; + integer = strtoull(value, NULL, 0); + /* extra_args keeps default value, it should be replaced + * only in case of successful parsing of the 'value' arg + */ + if (errno == 0) + *(uint64_t *)extra_args = integer; + return -errno; } static struct rte_eth_dev * -virtio_user_eth_dev_alloc(const char *name) +virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev) { struct rte_eth_dev *eth_dev; struct rte_eth_dev_data *data; struct virtio_hw *hw; struct virtio_user_dev *dev; - eth_dev = rte_eth_dev_allocate(name); + eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*hw)); if (!eth_dev) { PMD_INIT_LOG(ERR, "cannot alloc rte_eth_dev"); return NULL; } data = eth_dev->data; - - hw = rte_zmalloc(NULL, sizeof(*hw), 0); - if (!hw) { - PMD_INIT_LOG(ERR, "malloc virtio_hw failed"); - rte_eth_dev_release_port(eth_dev); - return NULL; - } + hw = eth_dev->data->dev_private; dev = rte_zmalloc(NULL, sizeof(*dev), 0); if (!dev) { PMD_INIT_LOG(ERR, "malloc virtio_user_dev failed"); rte_eth_dev_release_port(eth_dev); - rte_free(hw); return NULL; } - hw->vtpci_ops = &virtio_user_ops; - hw->use_msix = 0; + hw->port_id = data->port_id; + dev->port_id = data->port_id; + virtio_hw_internal[hw->port_id].vtpci_ops = &virtio_user_ops; + /* + * MSIX is required to enable LSC (see virtio_init_device). + * Here just pretend that we support msix. + */ + hw->use_msix = 1; hw->modern = 0; + hw->use_vec_rx = 0; + hw->use_vec_tx = 0; + hw->use_inorder_rx = 0; + hw->use_inorder_tx = 0; hw->virtio_user_dev = dev; - data->dev_private = hw; - data->numa_node = SOCKET_ID_ANY; - data->kdrv = RTE_KDRV_NONE; - data->dev_flags = RTE_ETH_DEV_DETACHABLE; - eth_dev->pci_dev = NULL; - eth_dev->driver = NULL; return eth_dev; } +static void +virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev) +{ + struct rte_eth_dev_data *data = eth_dev->data; + struct virtio_hw *hw = data->dev_private; + + rte_free(hw->virtio_user_dev); + rte_eth_dev_release_port(eth_dev); +} + /* Dev initialization routine. Invoked once for each virtio vdev at - * EAL init time, see rte_eal_dev_init(). + * EAL init time, see rte_bus_probe(). * Returns 0 on success. */ static int -virtio_user_pmd_devinit(const char *name, const char *params) +virtio_user_pmd_probe(struct rte_vdev_device *dev) { struct rte_kvargs *kvlist = NULL; struct rte_eth_dev *eth_dev; @@ -327,40 +579,74 @@ virtio_user_pmd_devinit(const char *name, const char *params) uint64_t queues = VIRTIO_USER_DEF_Q_NUM; uint64_t cq = VIRTIO_USER_DEF_CQ_EN; uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ; + uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE; + uint64_t mrg_rxbuf = 1; + uint64_t in_order = 1; + uint64_t packed_vq = 0; + uint64_t vectorized = 0; char *path = NULL; + char *ifname = NULL; char *mac_addr = NULL; int ret = -1; - if (!params || params[0] == '\0') { - PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user", - VIRTIO_USER_ARG_QUEUE_SIZE); - goto end; + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + const char *name = rte_vdev_device_name(dev); + eth_dev = rte_eth_dev_attach_secondary(name); + if (!eth_dev) { + PMD_INIT_LOG(ERR, "Failed to probe %s", name); + return -1; + } + + if (eth_virtio_dev_init(eth_dev) < 0) { + PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); + rte_eth_dev_release_port(eth_dev); + return -1; + } + + eth_dev->dev_ops = &virtio_user_secondary_eth_dev_ops; + eth_dev->device = &dev->device; + rte_eth_dev_probing_finish(eth_dev); + return 0; } - kvlist = rte_kvargs_parse(params, valid_args); + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args); if (!kvlist) { PMD_INIT_LOG(ERR, "error when parsing param"); goto end; } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PATH) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH, - &get_string_arg, &path); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH, + &get_string_arg, &path) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_PATH); goto end; } } else { - PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user\n", - VIRTIO_USER_ARG_QUEUE_SIZE); + PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user", + VIRTIO_USER_ARG_PATH); goto end; } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) { + if (is_vhost_user_by_type(path)) { + PMD_INIT_LOG(ERR, + "arg %s applies only to vhost-kernel backend", + VIRTIO_USER_ARG_INTERFACE_NAME); + goto end; + } + + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME, + &get_string_arg, &ifname) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_INTERFACE_NAME); + goto end; + } + } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC, - &get_string_arg, &mac_addr); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC, + &get_string_arg, &mac_addr) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_MAC); goto end; @@ -368,9 +654,8 @@ virtio_user_pmd_devinit(const char *name, const char *params) } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE, - &get_integer_arg, &queue_size); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE, + &get_integer_arg, &queue_size) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_QUEUE_SIZE); goto end; @@ -378,19 +663,26 @@ virtio_user_pmd_devinit(const char *name, const char *params) } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUES_NUM) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM, - &get_integer_arg, &queues); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM, + &get_integer_arg, &queues) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_QUEUES_NUM); goto end; } } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE, + &get_integer_arg, &server_mode) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_SERVER_MODE); + goto end; + } + } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM, - &get_integer_arg, &cq); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM, + &get_integer_arg, &cq) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_CQ_NUM); goto end; @@ -399,12 +691,55 @@ virtio_user_pmd_devinit(const char *name, const char *params) cq = 1; } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PACKED_VQ) == 1) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PACKED_VQ, + &get_integer_arg, &packed_vq) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_PACKED_VQ); + goto end; + } + } + + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED, + &get_integer_arg, &vectorized) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_VECTORIZED); + goto end; + } + } + if (queues > 1 && cq == 0) { PMD_INIT_LOG(ERR, "multi-q requires ctrl-q"); goto end; } - eth_dev = virtio_user_eth_dev_alloc(name); + if (queues > VIRTIO_MAX_VIRTQUEUE_PAIRS) { + PMD_INIT_LOG(ERR, "arg %s %" PRIu64 " exceeds the limit %u", + VIRTIO_USER_ARG_QUEUES_NUM, queues, + VIRTIO_MAX_VIRTQUEUE_PAIRS); + goto end; + } + + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MRG_RXBUF) == 1) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MRG_RXBUF, + &get_integer_arg, &mrg_rxbuf) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_MRG_RXBUF); + goto end; + } + } + + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_IN_ORDER) == 1) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_IN_ORDER, + &get_integer_arg, &in_order) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_IN_ORDER); + goto end; + } + } + + eth_dev = virtio_user_eth_dev_alloc(dev); if (!eth_dev) { PMD_INIT_LOG(ERR, "virtio_user fails to alloc device"); goto end; @@ -412,14 +747,35 @@ virtio_user_pmd_devinit(const char *name, const char *params) hw = eth_dev->data->dev_private; if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq, - queue_size, mac_addr) < 0) + queue_size, mac_addr, &ifname, server_mode, + mrg_rxbuf, in_order, packed_vq) < 0) { + PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); + virtio_user_eth_dev_free(eth_dev); goto end; + } - /* previously called by rte_eal_pci_probe() for physical dev */ + /* previously called by pci probing for physical dev */ if (eth_virtio_dev_init(eth_dev) < 0) { PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); + virtio_user_eth_dev_free(eth_dev); goto end; } + + if (vectorized) { + if (packed_vq) { +#if defined(CC_AVX512_SUPPORT) + hw->use_vec_rx = 1; + hw->use_vec_tx = 1; +#else + PMD_INIT_LOG(INFO, + "building environment do not support packed ring vectorized"); +#endif + } else { + hw->use_vec_rx = 1; + } + } + + rte_eth_dev_probing_finish(eth_dev); ret = 0; end: @@ -429,48 +785,53 @@ end: free(path); if (mac_addr) free(mac_addr); + if (ifname) + free(ifname); return ret; } -/** Called by rte_eth_dev_detach() */ static int -virtio_user_pmd_devuninit(const char *name) +virtio_user_pmd_remove(struct rte_vdev_device *vdev) { + const char *name; struct rte_eth_dev *eth_dev; - struct virtio_hw *hw; - struct virtio_user_dev *dev; - if (!name) + if (!vdev) return -EINVAL; - PMD_DRV_LOG(INFO, "Un-Initializing %s\n", name); + name = rte_vdev_device_name(vdev); + PMD_DRV_LOG(INFO, "Un-Initializing %s", name); eth_dev = rte_eth_dev_allocated(name); + /* Port has already been released by close. */ if (!eth_dev) - return -ENODEV; + return 0; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return rte_eth_dev_release_port(eth_dev); /* make sure the device is stopped, queues freed */ rte_eth_dev_close(eth_dev->data->port_id); - hw = eth_dev->data->dev_private; - dev = hw->virtio_user_dev; - virtio_user_dev_uninit(dev); - - rte_free(eth_dev->data->dev_private); - rte_free(eth_dev->data); - rte_eth_dev_release_port(eth_dev); - return 0; } static struct rte_vdev_driver virtio_user_driver = { - .init = virtio_user_pmd_devinit, - .uninit = virtio_user_pmd_devuninit, + .probe = virtio_user_pmd_probe, + .remove = virtio_user_pmd_remove, }; -DRIVER_REGISTER_VDEV(net_virtio_user, virtio_user_driver); -DRIVER_REGISTER_PARAM_STRING(net_virtio_user, +RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver); +RTE_PMD_REGISTER_ALIAS(net_virtio_user, virtio_user); +RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user, "path= " "mac= " "cq= " "queue_size= " - "queues="); + "queues= " + "iface= " + "server=<0|1> " + "mrg_rxbuf=<0|1> " + "in_order=<0|1> " + "packed_vq=<0|1> " + "speed= " + "vectorized=<0|1>");