X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_user_ethdev.c;h=40345193e6d3a7d5360f2f37011fe8b606485334;hb=5971ce5e2a591adcbadbe0de203985ab3ac70232;hp=c01f45cab32d57ece774be32eb77d46df9674a73;hpb=48a4464029a7f76dfb2c1f09146a391917b075e5;p=dpdk.git diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c index c01f45cab3..40345193e6 100644 --- a/drivers/net/virtio/virtio_user_ethdev.c +++ b/drivers/net/virtio/virtio_user_ethdev.c @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include #include #include @@ -13,6 +16,7 @@ #include #include #include +#include #include "virtio_ethdev.h" #include "virtio_logs.h" @@ -20,22 +24,67 @@ #include "virtqueue.h" #include "virtio_rxtx.h" #include "virtio_user/virtio_user_dev.h" +#include "virtio_user/vhost.h" #define virtio_user_get_dev(hw) \ ((struct virtio_user_dev *)(hw)->virtio_user_dev) +static void +virtio_user_reset_queues_packed(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw = dev->data->dev_private; + struct virtnet_rx *rxvq; + struct virtnet_tx *txvq; + uint16_t i; + + /* Add lock to avoid queue contention. */ + rte_spinlock_lock(&hw->state_lock); + hw->started = 0; + + /* + * Waitting for datapath to complete before resetting queues. + * 1 ms should be enough for the ongoing Tx/Rx function to finish. + */ + rte_delay_ms(1); + + /* Vring reset for each Tx queue and Rx queue. */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxvq = dev->data->rx_queues[i]; + virtqueue_rxvq_reset_packed(rxvq->vq); + virtio_dev_rx_queue_setup_finish(dev, i); + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txvq = dev->data->tx_queues[i]; + virtqueue_txvq_reset_packed(txvq->vq); + } + + hw->started = 1; + rte_spinlock_unlock(&hw->state_lock); +} + + static int virtio_user_server_reconnect(struct virtio_user_dev *dev) { - int ret; - int connectfd; + int ret, connectfd, old_status; struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id]; + struct virtio_hw *hw = eth_dev->data->dev_private; + uint64_t protocol_features; connectfd = accept(dev->listenfd, NULL, NULL); if (connectfd < 0) return -1; dev->vhostfd = connectfd; + old_status = vtpci_get_status(hw); + + vtpci_reset(hw); + + vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); + + vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); + if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES, &dev->device_features) < 0) { PMD_INIT_LOG(ERR, "get_features failed: %s", @@ -43,6 +92,28 @@ virtio_user_server_reconnect(struct virtio_user_dev *dev) return -1; } + if (dev->device_features & + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) { + if (dev->ops->send_request(dev, + VHOST_USER_GET_PROTOCOL_FEATURES, + &protocol_features)) + return -1; + + /* Offer VHOST_USER_PROTOCOL_F_STATUS */ + dev->protocol_features |= + (1ULL << VHOST_USER_PROTOCOL_F_STATUS); + dev->protocol_features &= protocol_features; + + if (dev->ops->send_request(dev, + VHOST_USER_SET_PROTOCOL_FEATURES, + &dev->protocol_features)) + return -1; + + if (!(dev->protocol_features & + (1ULL << VHOST_USER_PROTOCOL_F_MQ))) + dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); + } + dev->device_features |= dev->frontend_features; /* umask vhost-user unsupported features */ @@ -50,8 +121,19 @@ virtio_user_server_reconnect(struct virtio_user_dev *dev) dev->features &= dev->device_features; - ret = virtio_user_start_device(dev); - if (ret < 0) + /* For packed ring, resetting queues is required in reconnection. */ + if (vtpci_packed_queue(hw) && + (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { + PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" + " when packed ring reconnecting."); + virtio_user_reset_queues_packed(eth_dev); + } + + vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); + + /* Start the device */ + vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); + if (!dev->started) return -1; if (dev->queue_pairs > 1) { @@ -99,6 +181,11 @@ virtio_user_delayed_handler(void *param) if (dev->vhostfd >= 0) { close(dev->vhostfd); dev->vhostfd = -1; + /* Until the featuers are negotiated again, don't assume + * the backend supports VHOST_USER_PROTOCOL_F_STATUS + */ + dev->protocol_features &= + ~(1ULL << VHOST_USER_PROTOCOL_F_STATUS); } eth_dev->intr_handle->fd = dev->listenfd; rte_intr_callback_register(eth_dev->intr_handle, @@ -118,8 +205,8 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, struct virtio_user_dev *dev = virtio_user_get_dev(hw); if (offset == offsetof(struct virtio_net_config, mac) && - length == ETHER_ADDR_LEN) { - for (i = 0; i < ETHER_ADDR_LEN; ++i) + length == RTE_ETHER_ADDR_LEN) { + for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) ((uint8_t *)dst)[i] = dev->mac_addr[i]; return; } @@ -139,7 +226,7 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, } r = recv(dev->vhostfd, buf, 128, MSG_PEEK); if (r == 0 || (r < 0 && errno != EAGAIN)) { - dev->status &= (~VIRTIO_NET_S_LINK_UP); + dev->net_status &= (~VIRTIO_NET_S_LINK_UP); PMD_DRV_LOG(ERR, "virtio-user port %u is down", hw->port_id); @@ -151,7 +238,7 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, virtio_user_delayed_handler, (void *)hw); } else { - dev->status |= VIRTIO_NET_S_LINK_UP; + dev->net_status |= VIRTIO_NET_S_LINK_UP; } if (fcntl(dev->vhostfd, F_SETFL, flags & ~O_NONBLOCK) == -1) { @@ -159,12 +246,12 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, return; } } else if (dev->is_server) { - dev->status &= (~VIRTIO_NET_S_LINK_UP); + dev->net_status &= (~VIRTIO_NET_S_LINK_UP); if (virtio_user_server_reconnect(dev) >= 0) - dev->status |= VIRTIO_NET_S_LINK_UP; + dev->net_status |= VIRTIO_NET_S_LINK_UP; } - *(uint16_t *)dst = dev->status; + *(uint16_t *)dst = dev->net_status; } if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs)) @@ -179,8 +266,8 @@ virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset, struct virtio_user_dev *dev = virtio_user_get_dev(hw); if ((offset == offsetof(struct virtio_net_config, mac)) && - (length == ETHER_ADDR_LEN)) - for (i = 0; i < ETHER_ADDR_LEN; ++i) + (length == RTE_ETHER_ADDR_LEN)) + for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) dev->mac_addr[i] = ((const uint8_t *)src)[i]; else PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d", @@ -200,12 +287,17 @@ static void virtio_user_set_status(struct virtio_hw *hw, uint8_t status) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); + uint8_t old_status = dev->status; + if (status & VIRTIO_CONFIG_STATUS_FEATURES_OK && + ~old_status & VIRTIO_CONFIG_STATUS_FEATURES_OK) + virtio_user_dev_set_features(dev); if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) virtio_user_start_device(dev); else if (status == VIRTIO_CONFIG_STATUS_RESET) virtio_user_reset(hw); - dev->status = status; + + virtio_user_dev_set_status(dev, status); } static uint8_t @@ -213,6 +305,8 @@ virtio_user_get_status(struct virtio_hw *hw) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); + virtio_user_dev_update_status(dev); + return dev->status; } @@ -274,7 +368,6 @@ virtio_user_get_queue_num(struct virtio_hw *hw, uint16_t queue_id __rte_unused) static void virtio_user_setup_queue_packed(struct virtqueue *vq, struct virtio_user_dev *dev) - { uint16_t queue_idx = vq->vq_queue_index; struct vring_packed *vring; @@ -291,19 +384,14 @@ virtio_user_setup_queue_packed(struct virtqueue *vq, sizeof(struct vring_packed_desc_event), VIRTIO_PCI_VRING_ALIGN); vring->num = vq->vq_nentries; - vring->desc_packed = - (void *)(uintptr_t)desc_addr; - vring->driver_event = - (void *)(uintptr_t)avail_addr; - vring->device_event = - (void *)(uintptr_t)used_addr; + vring->desc = (void *)(uintptr_t)desc_addr; + vring->driver = (void *)(uintptr_t)avail_addr; + vring->device = (void *)(uintptr_t)used_addr; dev->packed_queues[queue_idx].avail_wrap_counter = true; dev->packed_queues[queue_idx].used_wrap_counter = true; - for (i = 0; i < vring->num; i++) { - vring->desc_packed[i].flags = VRING_DESC_F_USED(1) | - VRING_DESC_F_AVAIL(1); - } + for (i = 0; i < vring->num; i++) + vring->desc[i].flags = 0; } static void @@ -411,6 +499,10 @@ static const char *valid_args[] = { VIRTIO_USER_ARG_IN_ORDER, #define VIRTIO_USER_ARG_PACKED_VQ "packed_vq" VIRTIO_USER_ARG_PACKED_VQ, +#define VIRTIO_USER_ARG_SPEED "speed" + VIRTIO_USER_ARG_SPEED, +#define VIRTIO_USER_ARG_VECTORIZED "vectorized" + VIRTIO_USER_ARG_VECTORIZED, NULL }; @@ -438,15 +530,72 @@ static int get_integer_arg(const char *key __rte_unused, const char *value, void *extra_args) { + uint64_t integer = 0; if (!value || !extra_args) return -EINVAL; + errno = 0; + integer = strtoull(value, NULL, 0); + /* extra_args keeps default value, it should be replaced + * only in case of successful parsing of the 'value' arg + */ + if (errno == 0) + *(uint64_t *)extra_args = integer; + return -errno; +} - *(uint64_t *)extra_args = strtoull(value, NULL, 0); +static uint32_t +vdpa_dynamic_major_num(void) +{ + FILE *fp; + char *line = NULL; + size_t size; + char name[11]; + bool found = false; + uint32_t num; + + fp = fopen("/proc/devices", "r"); + if (fp == NULL) { + PMD_INIT_LOG(ERR, "Cannot open /proc/devices: %s", + strerror(errno)); + return UNNAMED_MAJOR; + } - return 0; + while (getline(&line, &size, fp) > 0) { + char *stripped = line + strspn(line, " "); + if ((sscanf(stripped, "%u %10s", &num, name) == 2) && + (strncmp(name, "vhost-vdpa", 10) == 0)) { + found = true; + break; + } + } + fclose(fp); + return found ? num : UNNAMED_MAJOR; } -static struct rte_vdev_driver virtio_user_driver; +static enum virtio_user_backend_type +virtio_user_backend_type(const char *path) +{ + struct stat sb; + + if (stat(path, &sb) == -1) { + if (errno == ENOENT) + return VIRTIO_USER_BACKEND_VHOST_USER; + + PMD_INIT_LOG(ERR, "Stat fails: %s (%s)\n", path, + strerror(errno)); + return VIRTIO_USER_BACKEND_UNKNOWN; + } + + if (S_ISSOCK(sb.st_mode)) { + return VIRTIO_USER_BACKEND_VHOST_USER; + } else if (S_ISCHR(sb.st_mode)) { + if (major(sb.st_rdev) == MISC_MAJOR) + return VIRTIO_USER_BACKEND_VHOST_KERNEL; + if (major(sb.st_rdev) == vdpa_dynamic_major_num()) + return VIRTIO_USER_BACKEND_VHOST_VDPA; + } + return VIRTIO_USER_BACKEND_UNKNOWN; +} static struct rte_eth_dev * virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev) @@ -481,7 +630,8 @@ virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev) */ hw->use_msix = 1; hw->modern = 0; - hw->use_simple_rx = 0; + hw->use_vec_rx = 0; + hw->use_vec_tx = 0; hw->use_inorder_rx = 0; hw->use_inorder_tx = 0; hw->virtio_user_dev = dev; @@ -508,6 +658,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) struct rte_kvargs *kvlist = NULL; struct rte_eth_dev *eth_dev; struct virtio_hw *hw; + enum virtio_user_backend_type backend_type = VIRTIO_USER_BACKEND_UNKNOWN; uint64_t queues = VIRTIO_USER_DEF_Q_NUM; uint64_t cq = VIRTIO_USER_DEF_CQ_EN; uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ; @@ -515,11 +666,32 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) uint64_t mrg_rxbuf = 1; uint64_t in_order = 1; uint64_t packed_vq = 0; + uint64_t vectorized = 0; char *path = NULL; char *ifname = NULL; char *mac_addr = NULL; int ret = -1; + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + const char *name = rte_vdev_device_name(dev); + eth_dev = rte_eth_dev_attach_secondary(name); + if (!eth_dev) { + PMD_INIT_LOG(ERR, "Failed to probe %s", name); + return -1; + } + + if (eth_virtio_dev_init(eth_dev) < 0) { + PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); + rte_eth_dev_release_port(eth_dev); + return -1; + } + + eth_dev->dev_ops = &virtio_user_secondary_eth_dev_ops; + eth_dev->device = &dev->device; + rte_eth_dev_probing_finish(eth_dev); + return 0; + } + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args); if (!kvlist) { PMD_INIT_LOG(ERR, "error when parsing param"); @@ -539,8 +711,18 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) goto end; } + backend_type = virtio_user_backend_type(path); + if (backend_type == VIRTIO_USER_BACKEND_UNKNOWN) { + PMD_INIT_LOG(ERR, + "unable to determine backend type for path %s", + path); + goto end; + } + PMD_INIT_LOG(INFO, "Backend type detected: %s", + virtio_user_backend_strings[backend_type]); + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) { - if (is_vhost_user_by_type(path)) { + if (backend_type != VIRTIO_USER_BACKEND_VHOST_KERNEL) { PMD_INIT_LOG(ERR, "arg %s applies only to vhost-kernel backend", VIRTIO_USER_ARG_INTERFACE_NAME); @@ -611,6 +793,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) } } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED, + &get_integer_arg, &vectorized) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_VECTORIZED); + goto end; + } + } + if (queues > 1 && cq == 0) { PMD_INIT_LOG(ERR, "multi-q requires ctrl-q"); goto end; @@ -641,42 +832,42 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) } } - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - struct virtio_user_dev *vu_dev; - - eth_dev = virtio_user_eth_dev_alloc(dev); - if (!eth_dev) { - PMD_INIT_LOG(ERR, "virtio_user fails to alloc device"); - goto end; - } - - hw = eth_dev->data->dev_private; - vu_dev = virtio_user_get_dev(hw); - if (server_mode == 1) - vu_dev->is_server = true; - else - vu_dev->is_server = false; - if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq, - queue_size, mac_addr, &ifname, mrg_rxbuf, - in_order, packed_vq) < 0) { - PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); - virtio_user_eth_dev_free(eth_dev); - goto end; - } + eth_dev = virtio_user_eth_dev_alloc(dev); + if (!eth_dev) { + PMD_INIT_LOG(ERR, "virtio_user fails to alloc device"); + goto end; + } - } else { - eth_dev = rte_eth_dev_attach_secondary(rte_vdev_device_name(dev)); - if (!eth_dev) - goto end; + hw = eth_dev->data->dev_private; + if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq, + queue_size, mac_addr, &ifname, server_mode, + mrg_rxbuf, in_order, packed_vq, backend_type) < 0) { + PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); + virtio_user_eth_dev_free(eth_dev); + goto end; } - /* previously called by rte_pci_probe() for physical dev */ + /* previously called by pci probing for physical dev */ if (eth_virtio_dev_init(eth_dev) < 0) { PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); virtio_user_eth_dev_free(eth_dev); goto end; } + if (vectorized) { + if (packed_vq) { +#if defined(CC_AVX512_SUPPORT) + hw->use_vec_rx = 1; + hw->use_vec_tx = 1; +#else + PMD_INIT_LOG(INFO, + "building environment do not support packed ring vectorized"); +#endif + } else { + hw->use_vec_rx = 1; + } + } + rte_eth_dev_probing_finish(eth_dev); ret = 0; @@ -697,8 +888,6 @@ virtio_user_pmd_remove(struct rte_vdev_device *vdev) { const char *name; struct rte_eth_dev *eth_dev; - struct virtio_hw *hw; - struct virtio_user_dev *dev; if (!vdev) return -EINVAL; @@ -706,17 +895,65 @@ virtio_user_pmd_remove(struct rte_vdev_device *vdev) name = rte_vdev_device_name(vdev); PMD_DRV_LOG(INFO, "Un-Initializing %s", name); eth_dev = rte_eth_dev_allocated(name); + /* Port has already been released by close. */ if (!eth_dev) - return -ENODEV; + return 0; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return rte_eth_dev_release_port(eth_dev); /* make sure the device is stopped, queues freed */ - rte_eth_dev_close(eth_dev->data->port_id); + return rte_eth_dev_close(eth_dev->data->port_id); +} - hw = eth_dev->data->dev_private; +static int virtio_user_pmd_dma_map(struct rte_vdev_device *vdev, void *addr, + uint64_t iova, size_t len) +{ + const char *name; + struct rte_eth_dev *eth_dev; + struct virtio_user_dev *dev; + struct virtio_hw *hw; + + if (!vdev) + return -EINVAL; + + name = rte_vdev_device_name(vdev); + eth_dev = rte_eth_dev_allocated(name); + /* Port has already been released by close. */ + if (!eth_dev) + return 0; + + hw = (struct virtio_hw *)eth_dev->data->dev_private; dev = hw->virtio_user_dev; - virtio_user_dev_uninit(dev); - rte_eth_dev_release_port(eth_dev); + if (dev->ops->dma_map) + return dev->ops->dma_map(dev, addr, iova, len); + + return 0; +} + +static int virtio_user_pmd_dma_unmap(struct rte_vdev_device *vdev, void *addr, + uint64_t iova, size_t len) +{ + const char *name; + struct rte_eth_dev *eth_dev; + struct virtio_user_dev *dev; + struct virtio_hw *hw; + + if (!vdev) + return -EINVAL; + + name = rte_vdev_device_name(vdev); + eth_dev = rte_eth_dev_allocated(name); + /* Port has already been released by close. */ + if (!eth_dev) + return 0; + + hw = (struct virtio_hw *)eth_dev->data->dev_private; + dev = hw->virtio_user_dev; + + if (dev->ops->dma_unmap) + return dev->ops->dma_unmap(dev, addr, iova, len); return 0; } @@ -724,6 +961,8 @@ virtio_user_pmd_remove(struct rte_vdev_device *vdev) static struct rte_vdev_driver virtio_user_driver = { .probe = virtio_user_pmd_probe, .remove = virtio_user_pmd_remove, + .dma_map = virtio_user_pmd_dma_map, + .dma_unmap = virtio_user_pmd_dma_unmap, }; RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver); @@ -738,4 +977,6 @@ RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user, "server=<0|1> " "mrg_rxbuf=<0|1> " "in_order=<0|1> " - "packed_vq=<0|1>"); + "packed_vq=<0|1> " + "speed= " + "vectorized=<0|1>");