X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_ethdev.c;h=6c233b75baa96ba05e9f60717926e55cdd1d3e1e;hb=c1d4e9d37abdc6c07a05f7d96928e624fea9ebb5;hp=0a69a4db11c41f520bbddf6d21946e77e5e14e96;hpb=4710e16a4a7b53c9f2cf38e6f6af945e9af59c26;p=dpdk.git diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index 0a69a4db11..6c233b75ba 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -40,7 +41,7 @@ static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev); static int virtio_dev_configure(struct rte_eth_dev *dev); static int virtio_dev_start(struct rte_eth_dev *dev); -static void virtio_dev_stop(struct rte_eth_dev *dev); +static int virtio_dev_stop(struct rte_eth_dev *dev); static int virtio_dev_promiscuous_enable(struct rte_eth_dev *dev); static int virtio_dev_promiscuous_disable(struct rte_eth_dev *dev); static int virtio_dev_allmulticast_enable(struct rte_eth_dev *dev); @@ -85,9 +86,6 @@ static int virtio_dev_queue_stats_mapping_set( uint8_t stat_idx, uint8_t is_rx); -int virtio_logtype_init; -int virtio_logtype_driver; - static void virtio_notify_peers(struct rte_eth_dev *dev); static void virtio_ack_link_announce(struct rte_eth_dev *dev); @@ -290,13 +288,10 @@ virtio_send_command_split(struct virtnet_ctl *cvq, virtqueue_notify(vq); - rte_rmb(); - while (VIRTQUEUE_NUSED(vq) == 0) { - rte_rmb(); + while (virtqueue_nused(vq) == 0) usleep(100); - } - while (VIRTQUEUE_NUSED(vq)) { + while (virtqueue_nused(vq)) { uint32_t idx, desc_idx, used_idx; struct vring_used_elem *uep; @@ -615,10 +610,9 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) txr = hdr_mz->addr; memset(txr, 0, vq_size * sizeof(*txr)); for (i = 0; i < vq_size; i++) { - struct vring_desc *start_dp = txr[i].tx_indir; - /* first indirect descriptor is always the tx header */ if (!vtpci_packed_queue(hw)) { + struct vring_desc *start_dp = txr[i].tx_indir; vring_desc_init_split(start_dp, RTE_DIM(txr[i].tx_indir)); start_dp->addr = txvq->virtio_net_hdr_mem @@ -627,6 +621,16 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) tx_hdr); start_dp->len = hw->vtnet_hdr_size; start_dp->flags = VRING_DESC_F_NEXT; + } else { + struct vring_packed_desc *start_dp = + txr[i].tx_packed_indir; + vring_desc_init_indirect_packed(start_dp, + RTE_DIM(txr[i].tx_packed_indir)); + start_dp->addr = txvq->virtio_net_hdr_mem + + i * sizeof(*txr) + + offsetof(struct virtio_tx_region, + tx_hdr); + start_dp->len = hw->vtnet_hdr_size; } } } @@ -710,16 +714,18 @@ virtio_alloc_queues(struct rte_eth_dev *dev) static void virtio_queues_unbind_intr(struct rte_eth_dev *dev); -static void +static int virtio_dev_close(struct rte_eth_dev *dev) { struct virtio_hw *hw = dev->data->dev_private; struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; PMD_INIT_LOG(DEBUG, "virtio_dev_close"); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; if (!hw->opened) - return; + return 0; hw->opened = false; /* reset the NIC */ @@ -749,6 +755,8 @@ virtio_dev_close(struct rte_eth_dev *dev) if (!hw->modern) rte_pci_ioport_unmap(VTPCI_IO(hw)); } + + return 0; } static int @@ -926,7 +934,6 @@ static const struct eth_dev_ops virtio_eth_dev_ops = { .rx_queue_intr_enable = virtio_dev_rx_queue_intr_enable, .rx_queue_intr_disable = virtio_dev_rx_queue_intr_disable, .rx_queue_release = virtio_dev_queue_release, - .rx_descriptor_done = virtio_dev_rx_queue_done, .tx_queue_setup = virtio_dev_tx_queue_setup, .tx_queue_release = virtio_dev_queue_release, /* collect stats per queue */ @@ -1361,12 +1368,13 @@ virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features) PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64, hw->guest_features); - if (hw->modern) { - if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { - PMD_INIT_LOG(ERR, - "VIRTIO_F_VERSION_1 features is not enabled."); - return -1; - } + if (hw->modern && !vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { + PMD_INIT_LOG(ERR, + "VIRTIO_F_VERSION_1 features is not enabled."); + return -1; + } + + if (hw->modern || hw->virtio_user_dev) { vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) { PMD_INIT_LOG(ERR, @@ -1496,9 +1504,9 @@ virtio_interrupt_handler(void *param) if (isr & VIRTIO_PCI_ISR_CONFIG) { if (virtio_dev_link_update(dev, 0) == 0) - _rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_INTR_LSC, - NULL); + rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, + NULL); if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { vtpci_read_dev_config(hw, @@ -1523,9 +1531,12 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev) if (vtpci_packed_queue(hw)) { PMD_INIT_LOG(INFO, "virtio: using packed ring %s Tx path on port %u", - hw->use_inorder_tx ? "inorder" : "standard", + hw->use_vec_tx ? "vectorized" : "standard", eth_dev->data->port_id); - eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed; + if (hw->use_vec_tx) + eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec; + else + eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed; } else { if (hw->use_inorder_tx) { PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u", @@ -1539,7 +1550,13 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev) } if (vtpci_packed_queue(hw)) { - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + if (hw->use_vec_rx) { + PMD_INIT_LOG(INFO, + "virtio: using packed ring vectorized Rx path on port %u", + eth_dev->data->port_id); + eth_dev->rx_pkt_burst = + &virtio_recv_pkts_packed_vec; + } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { PMD_INIT_LOG(INFO, "virtio: using packed ring mergeable buffer Rx path on port %u", eth_dev->data->port_id); @@ -1663,7 +1680,6 @@ virtio_configure_intr(struct rte_eth_dev *dev) return 0; } -#define SPEED_UNKNOWN 0xffffffff #define DUPLEX_UNKNOWN 0xff /* reset device and renegotiate features if needed */ static int @@ -1703,6 +1719,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) else eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC; + eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; + /* Setting up rx_header size for the device */ if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || vtpci_with_feature(hw, VIRTIO_F_VERSION_1) || @@ -1720,7 +1738,7 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); - if (hw->speed == SPEED_UNKNOWN) { + if (hw->speed == ETH_SPEED_NUM_UNKNOWN) { if (vtpci_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) { config = &local_config; vtpci_read_dev_config(hw, @@ -1733,8 +1751,6 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) hw->duplex = config->duplex; } } - if (hw->speed == SPEED_UNKNOWN) - hw->speed = ETH_SPEED_NUM_10G; if (hw->duplex == DUPLEX_UNKNOWN) hw->duplex = ETH_LINK_FULL_DUPLEX; PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d", @@ -1886,7 +1902,7 @@ int eth_virtio_dev_init(struct rte_eth_dev *eth_dev) { struct virtio_hw *hw = eth_dev->data->dev_private; - uint32_t speed = SPEED_UNKNOWN; + uint32_t speed = ETH_SPEED_NUM_UNKNOWN; int vectorized = 0; int ret; @@ -1900,6 +1916,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) } eth_dev->dev_ops = &virtio_eth_dev_ops; + eth_dev->rx_descriptor_done = virtio_dev_rx_queue_done; if (rte_eal_process_type() == RTE_PROC_SECONDARY) { if (!hw->virtio_user_dev) { @@ -1918,11 +1935,6 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) if (ret < 0) return ret; hw->speed = speed; - /* - * Pass the information to the rte_eth_dev_close() that it should also - * release the private port resources. - */ - eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; /* Allocate memory for storing MAC addresses */ eth_dev->data->mac_addrs = rte_zmalloc("virtio", @@ -1952,8 +1964,17 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) goto err_virtio_init; if (vectorized) { - if (!vtpci_packed_queue(hw)) + if (!vtpci_packed_queue(hw)) { + hw->use_vec_rx = 1; + } else { +#if !defined(CC_AVX512_SUPPORT) + PMD_DRV_LOG(INFO, + "building environment do not support packed ring vectorized"); +#else hw->use_vec_rx = 1; + hw->use_vec_tx = 1; +#endif + } } hw->opened = true; @@ -1975,21 +1996,18 @@ err_vtpci_init: static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) { + int ret; PMD_INIT_FUNC_TRACE(); if (rte_eal_process_type() == RTE_PROC_SECONDARY) return 0; - virtio_dev_stop(eth_dev); + ret = virtio_dev_stop(eth_dev); virtio_dev_close(eth_dev); - eth_dev->dev_ops = NULL; - eth_dev->tx_pkt_burst = NULL; - eth_dev->rx_pkt_burst = NULL; - PMD_INIT_LOG(DEBUG, "dev_uninit completed"); - return 0; + return ret; } @@ -2023,6 +2041,8 @@ virtio_dev_speed_capa_get(uint32_t speed) return ETH_LINK_SPEED_56G; case ETH_SPEED_NUM_100G: return ETH_LINK_SPEED_100G; + case ETH_SPEED_NUM_200G: + return ETH_LINK_SPEED_200G; default: return 0; } @@ -2288,31 +2308,73 @@ virtio_dev_configure(struct rte_eth_dev *dev) return -EBUSY; } - if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) { - hw->use_inorder_tx = 1; - hw->use_inorder_rx = 1; - hw->use_vec_rx = 0; - } - if (vtpci_packed_queue(hw)) { +#if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT) + if ((hw->use_vec_rx || hw->use_vec_tx) && + (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) || + !vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) || + !vtpci_with_feature(hw, VIRTIO_F_VERSION_1) || + rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_512)) { + PMD_DRV_LOG(INFO, + "disabled packed ring vectorized path for requirements not met"); + hw->use_vec_rx = 0; + hw->use_vec_tx = 0; + } +#else hw->use_vec_rx = 0; - hw->use_inorder_rx = 0; - } + hw->use_vec_tx = 0; +#endif -#if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM - if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { - hw->use_vec_rx = 0; - } + if (hw->use_vec_rx) { + if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + PMD_DRV_LOG(INFO, + "disabled packed ring vectorized rx for mrg_rxbuf enabled"); + hw->use_vec_rx = 0; + } + + if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) { + PMD_DRV_LOG(INFO, + "disabled packed ring vectorized rx for TCP_LRO enabled"); + hw->use_vec_rx = 0; + } + } + } else { + if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) { + hw->use_inorder_tx = 1; + hw->use_inorder_rx = 1; + hw->use_vec_rx = 0; + } + + if (hw->use_vec_rx) { +#if defined RTE_ARCH_ARM + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { + PMD_DRV_LOG(INFO, + "disabled split ring vectorized path for requirement not met"); + hw->use_vec_rx = 0; + } #endif - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { - hw->use_vec_rx = 0; - } + if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + PMD_DRV_LOG(INFO, + "disabled split ring vectorized rx for mrg_rxbuf enabled"); + hw->use_vec_rx = 0; + } - if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM | - DEV_RX_OFFLOAD_TCP_LRO | - DEV_RX_OFFLOAD_VLAN_STRIP)) - hw->use_vec_rx = 0; + if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_TCP_LRO | + DEV_RX_OFFLOAD_VLAN_STRIP)) { + PMD_DRV_LOG(INFO, + "disabled split ring vectorized rx for offloading enabled"); + hw->use_vec_rx = 0; + } + + if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) { + PMD_DRV_LOG(INFO, + "disabled split ring vectorized rx, max SIMD bitwidth too low"); + hw->use_vec_rx = 0; + } + } + } return 0; } @@ -2460,7 +2522,7 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) /* * Stop device: disable interrupt and mark link down */ -static void +static int virtio_dev_stop(struct rte_eth_dev *dev) { struct virtio_hw *hw = dev->data->dev_private; @@ -2468,6 +2530,7 @@ virtio_dev_stop(struct rte_eth_dev *dev) struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; PMD_INIT_LOG(DEBUG, "stop"); + dev->data->dev_started = 0; rte_spinlock_lock(&hw->state_lock); if (!hw->started) @@ -2489,6 +2552,8 @@ virtio_dev_stop(struct rte_eth_dev *dev) rte_eth_linkstatus_set(dev, &link); out_unlock: rte_spinlock_unlock(&hw->state_lock); + + return 0; } static int @@ -2505,6 +2570,7 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet if (!hw->started) { link.link_status = ETH_LINK_DOWN; + link.link_speed = ETH_SPEED_NUM_NONE; } else if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { PMD_INIT_LOG(DEBUG, "Get link status from hw"); vtpci_read_dev_config(hw, @@ -2512,6 +2578,7 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet &status, sizeof(status)); if ((status & VIRTIO_NET_S_LINK_UP) == 0) { link.link_status = ETH_LINK_DOWN; + link.link_speed = ETH_SPEED_NUM_NONE; PMD_INIT_LOG(DEBUG, "Port %d is down", dev->data->port_id); } else { @@ -2609,13 +2676,5 @@ __rte_unused uint8_t is_rx) RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__); RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map); RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio-pci"); - -RTE_INIT(virtio_init_log) -{ - virtio_logtype_init = rte_log_register("pmd.net.virtio.init"); - if (virtio_logtype_init >= 0) - rte_log_set_level(virtio_logtype_init, RTE_LOG_NOTICE); - virtio_logtype_driver = rte_log_register("pmd.net.virtio.driver"); - if (virtio_logtype_driver >= 0) - rte_log_set_level(virtio_logtype_driver, RTE_LOG_NOTICE); -} +RTE_LOG_REGISTER(virtio_logtype_init, pmd.net.virtio.init, NOTICE); +RTE_LOG_REGISTER(virtio_logtype_driver, pmd.net.virtio.driver, NOTICE);