X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_ethdev.c;h=3ebc221f99c8ef0ac2d7967db7dfe5f07bdc8075;hb=09419f235e099ecb265a590778fe64a685a2a241;hp=57ae6c4e42aa2625c67263552b22c53b99289211;hpb=4ecce8356e9ef02cef747793ef93767665154892;p=dpdk.git diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index 57ae6c4e42..3ebc221f99 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,10 +36,6 @@ #include #include #include -#ifdef RTE_EXEC_ENV_LINUXAPP -#include -#include -#endif #include #include @@ -51,6 +47,7 @@ #include #include #include +#include #include #include @@ -60,9 +57,11 @@ #include "virtio_pci.h" #include "virtio_logs.h" #include "virtqueue.h" +#include "virtio_rxtx.h" static int eth_virtio_dev_init(struct rte_eth_dev *eth_dev); +static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev); static int virtio_dev_configure(struct rte_eth_dev *dev); static int virtio_dev_start(struct rte_eth_dev *dev); static void virtio_dev_stop(struct rte_eth_dev *dev); @@ -78,10 +77,10 @@ static int virtio_dev_link_update(struct rte_eth_dev *dev, static void virtio_set_hwaddr(struct virtio_hw *hw); static void virtio_get_hwaddr(struct virtio_hw *hw); -static void virtio_dev_rx_queue_release(__rte_unused void *rxq); -static void virtio_dev_tx_queue_release(__rte_unused void *txq); - -static void virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); +static void virtio_dev_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats); +static int virtio_dev_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstats *xstats, unsigned n); static void virtio_dev_stats_reset(struct rte_eth_dev *dev); static void virtio_dev_free_mbufs(struct rte_eth_dev *dev); static int virtio_vlan_filter_set(struct rte_eth_dev *dev, @@ -110,23 +109,47 @@ static const struct rte_pci_id pci_id_virtio_map[] = { { .vendor_id = 0, /* sentinel */ }, }; +struct rte_virtio_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + unsigned offset; +}; + +/* [rt]x_qX_ is prepended to the name string here */ +static const struct rte_virtio_xstats_name_off rte_virtio_q_stat_strings[] = { + {"good_packets", offsetof(struct virtqueue, packets)}, + {"good_bytes", offsetof(struct virtqueue, bytes)}, + {"errors", offsetof(struct virtqueue, errors)}, + {"multicast_packets", offsetof(struct virtqueue, multicast)}, + {"broadcast_packets", offsetof(struct virtqueue, broadcast)}, + {"undersize_packets", offsetof(struct virtqueue, size_bins[0])}, + {"size_64_packets", offsetof(struct virtqueue, size_bins[1])}, + {"size_65_127_packets", offsetof(struct virtqueue, size_bins[2])}, + {"size_128_255_packets", offsetof(struct virtqueue, size_bins[3])}, + {"size_256_511_packets", offsetof(struct virtqueue, size_bins[4])}, + {"size_512_1023_packets", offsetof(struct virtqueue, size_bins[5])}, + {"size_1024_1517_packets", offsetof(struct virtqueue, size_bins[6])}, + {"size_1518_max_packets", offsetof(struct virtqueue, size_bins[7])}, +}; + +#define VIRTIO_NB_Q_XSTATS (sizeof(rte_virtio_q_stat_strings) / \ + sizeof(rte_virtio_q_stat_strings[0])) + static int virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl, int *dlen, int pkt_num) { - uint16_t head = vq->vq_desc_head_idx, i; + uint32_t head, i; int k, sum = 0; virtio_net_ctrl_ack status = ~0; struct virtio_pmd_ctrl result; ctrl->status = status; - if (!vq->hw->cvq) { - PMD_INIT_LOG(ERR, - "%s(): Control queue is not supported.", - __func__); + if (!(vq && vq->hw->cvq)) { + PMD_INIT_LOG(ERR, "Control queue is not supported."); return -1; } + head = vq->vq_desc_head_idx; PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, " "vq->hw->cvq = %p vq = %p", @@ -237,43 +260,49 @@ virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues) return 0; } +void +virtio_dev_queue_release(struct virtqueue *vq) { + struct virtio_hw *hw; + + if (vq) { + hw = vq->hw; + hw->vtpci_ops->del_queue(hw, vq); + + rte_free(vq->sw_ring); + rte_free(vq); + } +} + int virtio_dev_queue_setup(struct rte_eth_dev *dev, int queue_type, uint16_t queue_idx, - uint16_t vtpci_queue_idx, + uint16_t vtpci_queue_idx, uint16_t nb_desc, unsigned int socket_id, struct virtqueue **pvq) { char vq_name[VIRTQUEUE_MAX_NAME_SZ]; const struct rte_memzone *mz; - uint16_t vq_size; - int size; + unsigned int vq_size, size; struct virtio_hw *hw = dev->data->dev_private; - struct virtqueue *vq = NULL; + struct virtqueue *vq = NULL; - /* Write the virtqueue index to the Queue Select Field */ - VIRTIO_WRITE_REG_2(hw, VIRTIO_PCI_QUEUE_SEL, vtpci_queue_idx); - PMD_INIT_LOG(DEBUG, "selecting queue: %d", vtpci_queue_idx); + PMD_INIT_LOG(DEBUG, "setting up queue: %u", vtpci_queue_idx); /* * Read the virtqueue size from the Queue Size field * Always power of 2 and if 0 virtqueue does not exist */ - vq_size = VIRTIO_READ_REG_2(hw, VIRTIO_PCI_QUEUE_NUM); - PMD_INIT_LOG(DEBUG, "vq_size: %d nb_desc:%d", vq_size, nb_desc); - if (nb_desc == 0) - nb_desc = vq_size; + vq_size = hw->vtpci_ops->get_queue_num(hw, vtpci_queue_idx); + PMD_INIT_LOG(DEBUG, "vq_size: %u nb_desc:%u", vq_size, nb_desc); if (vq_size == 0) { - PMD_INIT_LOG(ERR, "%s: virtqueue does not exist", __func__); + PMD_INIT_LOG(ERR, "virtqueue does not exist"); return -EINVAL; - } else if (!rte_is_power_of_2(vq_size)) { - PMD_INIT_LOG(ERR, "%s: virtqueue size is not powerof 2", __func__); + } + + if (!rte_is_power_of_2(vq_size)) { + PMD_INIT_LOG(ERR, "virtqueue size is not powerof 2"); return -EINVAL; - } else if (nb_desc != vq_size) { - PMD_INIT_LOG(ERR, "Warning: nb_desc(%d) is not equal to vq size (%d), fall to vq size", - nb_desc, vq_size); - nb_desc = vq_size; } if (queue_type == VTNET_RQ) { @@ -281,6 +310,9 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, dev->data->port_id, queue_idx); vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); + vq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", + (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) * + sizeof(vq->sw_ring[0]), RTE_CACHE_LINE_SIZE, socket_id); } else if (queue_type == VTNET_TQ) { snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d", dev->data->port_id, queue_idx); @@ -294,8 +326,13 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, RTE_CACHE_LINE_SIZE); } if (vq == NULL) { - PMD_INIT_LOG(ERR, "%s: Can not allocate virtqueue", __func__); - return (-ENOMEM); + PMD_INIT_LOG(ERR, "Can not allocate virtqueue"); + return -ENOMEM; + } + if (queue_type == VTNET_RQ && vq->sw_ring == NULL) { + PMD_INIT_LOG(ERR, "Can not allocate RX soft ring"); + rte_free(vq); + return -ENOMEM; } vq->hw = hw; @@ -303,7 +340,10 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, vq->queue_id = queue_idx; vq->vq_queue_index = vtpci_queue_idx; vq->vq_nentries = vq_size; - vq->vq_free_cnt = vq_size; + + if (nb_desc == 0 || nb_desc > vq_size) + nb_desc = vq_size; + vq->vq_free_cnt = nb_desc; /* * Reserve a memzone for vring elements @@ -315,8 +355,12 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, socket_id, 0, VIRTIO_PCI_VRING_ALIGN); if (mz == NULL) { - rte_free(vq); - return -ENOMEM; + if (rte_errno == EEXIST) + mz = rte_memzone_lookup(vq_name); + if (mz == NULL) { + rte_free(vq); + return -ENOMEM; + } } /* @@ -340,22 +384,47 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, vq->virtio_net_hdr_mem = 0; if (queue_type == VTNET_TQ) { + const struct rte_memzone *hdr_mz; + struct virtio_tx_region *txr; + unsigned int i; + /* * For each xmit packet, allocate a virtio_net_hdr + * and indirect ring elements */ snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d_hdrzone", - dev->data->port_id, queue_idx); - vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name, - vq_size * hw->vtnet_hdr_size, - socket_id, 0, RTE_CACHE_LINE_SIZE); - if (vq->virtio_net_hdr_mz == NULL) { - rte_free(vq); - return -ENOMEM; + dev->data->port_id, queue_idx); + hdr_mz = rte_memzone_reserve_aligned(vq_name, + vq_size * sizeof(*txr), + socket_id, 0, + RTE_CACHE_LINE_SIZE); + if (hdr_mz == NULL) { + if (rte_errno == EEXIST) + hdr_mz = rte_memzone_lookup(vq_name); + if (hdr_mz == NULL) { + rte_free(vq); + return -ENOMEM; + } } - vq->virtio_net_hdr_mem = - vq->virtio_net_hdr_mz->phys_addr; - memset(vq->virtio_net_hdr_mz->addr, 0, - vq_size * hw->vtnet_hdr_size); + vq->virtio_net_hdr_mz = hdr_mz; + vq->virtio_net_hdr_mem = hdr_mz->phys_addr; + + txr = hdr_mz->addr; + memset(txr, 0, vq_size * sizeof(*txr)); + for (i = 0; i < vq_size; i++) { + struct vring_desc *start_dp = txr[i].tx_indir; + + vring_desc_init(start_dp, RTE_DIM(txr[i].tx_indir)); + + /* first indirect descriptor is always the tx header */ + start_dp->addr = vq->virtio_net_hdr_mem + + i * sizeof(*txr) + + offsetof(struct virtio_tx_region, tx_hdr); + + start_dp->len = vq->hw->vtnet_hdr_size; + start_dp->flags = VRING_DESC_F_NEXT; + } + } else if (queue_type == VTNET_CQ) { /* Allocate a page for control vq command, data and status */ snprintf(vq_name, sizeof(vq_name), "port%d_cvq_hdrzone", @@ -363,20 +432,21 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name, PAGE_SIZE, socket_id, 0, RTE_CACHE_LINE_SIZE); if (vq->virtio_net_hdr_mz == NULL) { - rte_free(vq); - return -ENOMEM; + if (rte_errno == EEXIST) + vq->virtio_net_hdr_mz = + rte_memzone_lookup(vq_name); + if (vq->virtio_net_hdr_mz == NULL) { + rte_free(vq); + return -ENOMEM; + } } vq->virtio_net_hdr_mem = vq->virtio_net_hdr_mz->phys_addr; memset(vq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); } - /* - * Set guest physical address of the virtqueue - * in VIRTIO_PCI_QUEUE_PFN config register of device - */ - VIRTIO_WRITE_REG_4(hw, VIRTIO_PCI_QUEUE_PFN, - mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); + hw->vtpci_ops->setup_queue(hw, vq); + *pvq = vq; return 0; } @@ -386,13 +456,12 @@ virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx, uint32_t socket_id) { struct virtqueue *vq; - uint16_t nb_desc = 0; int ret; struct virtio_hw *hw = dev->data->dev_private; PMD_INIT_FUNC_TRACE(); ret = virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX, - vtpci_queue_idx, nb_desc, socket_id, &vq); + vtpci_queue_idx, 0, socket_id, &vq); if (ret < 0) { PMD_INIT_LOG(ERR, "control vq initialization failed"); return ret; @@ -402,6 +471,22 @@ virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx, return 0; } +static void +virtio_free_queues(struct rte_eth_dev *dev) +{ + unsigned int i; + + for (i = 0; i < dev->data->nb_rx_queues; i++) + virtio_dev_rx_queue_release(dev->data->rx_queues[i]); + + dev->data->nb_rx_queues = 0; + + for (i = 0; i < dev->data->nb_tx_queues; i++) + virtio_dev_tx_queue_release(dev->data->tx_queues[i]); + + dev->data->nb_tx_queues = 0; +} + static void virtio_dev_close(struct rte_eth_dev *dev) { @@ -410,12 +495,15 @@ virtio_dev_close(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "virtio_dev_close"); + if (hw->started == 1) + virtio_dev_stop(dev); + /* reset the NIC */ if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) vtpci_irq_config(hw, VIRTIO_MSI_NO_VECTOR); vtpci_reset(hw); - hw->started = 0; virtio_dev_free_mbufs(dev); + virtio_free_queues(dev); } static void @@ -426,6 +514,11 @@ virtio_dev_promiscuous_enable(struct rte_eth_dev *dev) int dlen[1]; int ret; + if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + PMD_INIT_LOG(INFO, "host does not support rx control\n"); + return; + } + ctrl.hdr.class = VIRTIO_NET_CTRL_RX; ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; ctrl.data[0] = 1; @@ -444,6 +537,11 @@ virtio_dev_promiscuous_disable(struct rte_eth_dev *dev) int dlen[1]; int ret; + if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + PMD_INIT_LOG(INFO, "host does not support rx control\n"); + return; + } + ctrl.hdr.class = VIRTIO_NET_CTRL_RX; ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; ctrl.data[0] = 0; @@ -462,6 +560,11 @@ virtio_dev_allmulticast_enable(struct rte_eth_dev *dev) int dlen[1]; int ret; + if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + PMD_INIT_LOG(INFO, "host does not support rx control\n"); + return; + } + ctrl.hdr.class = VIRTIO_NET_CTRL_RX; ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; ctrl.data[0] = 1; @@ -480,6 +583,11 @@ virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) int dlen[1]; int ret; + if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + PMD_INIT_LOG(INFO, "host does not support rx control\n"); + return; + } + ctrl.hdr.class = VIRTIO_NET_CTRL_RX; ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; ctrl.data[0] = 0; @@ -505,13 +613,13 @@ static const struct eth_dev_ops virtio_eth_dev_ops = { .dev_infos_get = virtio_dev_info_get, .stats_get = virtio_dev_stats_get, + .xstats_get = virtio_dev_xstats_get, .stats_reset = virtio_dev_stats_reset, + .xstats_reset = virtio_dev_stats_reset, .link_update = virtio_dev_link_update, .rx_queue_setup = virtio_dev_rx_queue_setup, - /* meaningfull only to multiple queue */ .rx_queue_release = virtio_dev_rx_queue_release, .tx_queue_setup = virtio_dev_tx_queue_setup, - /* meaningfull only to multiple queue */ .tx_queue_release = virtio_dev_tx_queue_release, /* collect stats per queue */ .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, @@ -562,7 +670,7 @@ virtio_dev_atomic_write_link_status(struct rte_eth_dev *dev, } static void -virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { unsigned i; @@ -599,6 +707,64 @@ virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed; } +static int +virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats, + unsigned n) +{ + unsigned i; + unsigned count = 0; + + unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_Q_XSTATS + + dev->data->nb_rx_queues * VIRTIO_NB_Q_XSTATS; + + if (n < nstats) + return nstats; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct virtqueue *rxvq = dev->data->rx_queues[i]; + + if (rxvq == NULL) + continue; + + unsigned t; + + for (t = 0; t < VIRTIO_NB_Q_XSTATS; t++) { + snprintf(xstats[count].name, sizeof(xstats[count].name), + "rx_q%u_%s", i, + rte_virtio_q_stat_strings[t].name); + xstats[count].value = *(uint64_t *)(((char *)rxvq) + + rte_virtio_q_stat_strings[t].offset); + count++; + } + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct virtqueue *txvq = dev->data->tx_queues[i]; + + if (txvq == NULL) + continue; + + unsigned t; + + for (t = 0; t < VIRTIO_NB_Q_XSTATS; t++) { + snprintf(xstats[count].name, sizeof(xstats[count].name), + "tx_q%u_%s", i, + rte_virtio_q_stat_strings[t].name); + xstats[count].value = *(uint64_t *)(((char *)txvq) + + rte_virtio_q_stat_strings[t].offset); + count++; + } + } + + return count; +} + +static void +virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + virtio_update_stats(dev, stats); +} + static void virtio_dev_stats_reset(struct rte_eth_dev *dev) { @@ -612,6 +778,9 @@ virtio_dev_stats_reset(struct rte_eth_dev *dev) txvq->packets = 0; txvq->bytes = 0; txvq->errors = 0; + txvq->multicast = 0; + txvq->broadcast = 0; + memset(txvq->size_bins, 0, sizeof(txvq->size_bins[0]) * 8); } for (i = 0; i < dev->data->nb_rx_queues; i++) { @@ -622,9 +791,10 @@ virtio_dev_stats_reset(struct rte_eth_dev *dev) rxvq->packets = 0; rxvq->bytes = 0; rxvq->errors = 0; + rxvq->multicast = 0; + rxvq->broadcast = 0; + memset(rxvq->size_bins, 0, sizeof(rxvq->size_bins[0]) * 8); } - - dev->data->rx_mbuf_alloc_failed = 0; } static void @@ -648,7 +818,7 @@ virtio_get_hwaddr(struct virtio_hw *hw) } } -static int +static void virtio_mac_table_set(struct virtio_hw *hw, const struct virtio_net_ctrl_mac *uc, const struct virtio_net_ctrl_mac *mc) @@ -656,6 +826,11 @@ virtio_mac_table_set(struct virtio_hw *hw, struct virtio_pmd_ctrl ctrl; int err, len[2]; + if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { + PMD_DRV_LOG(INFO, "host does not support mac table\n"); + return; + } + ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; @@ -668,8 +843,6 @@ virtio_mac_table_set(struct virtio_hw *hw, err = virtio_send_command(hw->cvq, &ctrl, len, 2); if (err != 0) PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err); - - return err; } static void @@ -773,32 +946,19 @@ virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) return virtio_send_command(hw->cvq, &ctrl, &len, 1); } -static void +static int virtio_negotiate_features(struct virtio_hw *hw) { - uint32_t host_features, mask; - - /* checksum offload not implemented */ - mask = VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM; - - /* TSO and LRO are only available when their corresponding - * checksum offload feature is also negotiated. - */ - mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 | VIRTIO_NET_F_HOST_ECN; - mask |= VIRTIO_NET_F_GUEST_TSO4 | VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN; - mask |= VTNET_LRO_FEATURES; - - /* not negotiating INDIRECT descriptor table support */ - mask |= VIRTIO_RING_F_INDIRECT_DESC; + uint64_t host_features; /* Prepare guest_features: feature that driver wants to support */ - hw->guest_features = VTNET_FEATURES & ~mask; - PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %x", + hw->guest_features = VIRTIO_PMD_GUEST_FEATURES; + PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64, hw->guest_features); /* Read device(host) feature bits */ - host_features = VIRTIO_READ_REG_4(hw, VIRTIO_PCI_HOST_FEATURES); - PMD_INIT_LOG(DEBUG, "host_features before negotiate = %x", + host_features = hw->vtpci_ops->get_features(hw); + PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64, host_features); /* @@ -806,263 +966,25 @@ virtio_negotiate_features(struct virtio_hw *hw) * guest feature bits. */ hw->guest_features = vtpci_negotiate_features(hw, host_features); - PMD_INIT_LOG(DEBUG, "features after negotiate = %x", + PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64, hw->guest_features); -} - -#ifdef RTE_EXEC_ENV_LINUXAPP -static int -parse_sysfs_value(const char *filename, unsigned long *val) -{ - FILE *f; - char buf[BUFSIZ]; - char *end = NULL; - - f = fopen(filename, "r"); - if (f == NULL) { - PMD_INIT_LOG(ERR, "%s(): cannot open sysfs value %s", - __func__, filename); - return -1; - } - - if (fgets(buf, sizeof(buf), f) == NULL) { - PMD_INIT_LOG(ERR, "%s(): cannot read sysfs value %s", - __func__, filename); - fclose(f); - return -1; - } - *val = strtoul(buf, &end, 0); - if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) { - PMD_INIT_LOG(ERR, "%s(): cannot parse sysfs value %s", - __func__, filename); - fclose(f); - return -1; - } - fclose(f); - return 0; -} -static int get_uio_dev(struct rte_pci_addr *loc, char *buf, unsigned int buflen, - unsigned int *uio_num) -{ - struct dirent *e; - DIR *dir; - char dirname[PATH_MAX]; - - /* depending on kernel version, uio can be located in uio/uioX - * or uio:uioX */ - snprintf(dirname, sizeof(dirname), - SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/uio", - loc->domain, loc->bus, loc->devid, loc->function); - dir = opendir(dirname); - if (dir == NULL) { - /* retry with the parent directory */ - snprintf(dirname, sizeof(dirname), - SYSFS_PCI_DEVICES "/" PCI_PRI_FMT, - loc->domain, loc->bus, loc->devid, loc->function); - dir = opendir(dirname); - - if (dir == NULL) { - PMD_INIT_LOG(ERR, "Cannot opendir %s", dirname); + if (hw->modern) { + if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { + PMD_INIT_LOG(ERR, + "VIRTIO_F_VERSION_1 features is not enabled."); return -1; } - } - - /* take the first file starting with "uio" */ - while ((e = readdir(dir)) != NULL) { - /* format could be uio%d ...*/ - int shortprefix_len = sizeof("uio") - 1; - /* ... or uio:uio%d */ - int longprefix_len = sizeof("uio:uio") - 1; - char *endptr; - - if (strncmp(e->d_name, "uio", 3) != 0) - continue; - - /* first try uio%d */ - errno = 0; - *uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); - if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { - snprintf(buf, buflen, "%s/uio%u", dirname, *uio_num); - break; - } - - /* then try uio:uio%d */ - errno = 0; - *uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); - if (errno == 0 && endptr != (e->d_name + longprefix_len)) { - snprintf(buf, buflen, "%s/uio:uio%u", dirname, - *uio_num); - break; - } - } - closedir(dir); - - /* No uio resource found */ - if (e == NULL) { - PMD_INIT_LOG(ERR, "Could not find uio resource"); - return -1; - } - - return 0; -} - -static int -virtio_has_msix(const struct rte_pci_addr *loc) -{ - DIR *d; - char dirname[PATH_MAX]; - - snprintf(dirname, sizeof(dirname), - SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/msi_irqs", - loc->domain, loc->bus, loc->devid, loc->function); - - d = opendir(dirname); - if (d) - closedir(d); - - return (d != NULL); -} - -/* Extract I/O port numbers from sysfs */ -static int virtio_resource_init_by_uio(struct rte_pci_device *pci_dev) -{ - char dirname[PATH_MAX]; - char filename[PATH_MAX]; - unsigned long start, size; - unsigned int uio_num; - - if (get_uio_dev(&pci_dev->addr, dirname, sizeof(dirname), &uio_num) < 0) - return -1; - - /* get portio size */ - snprintf(filename, sizeof(filename), - "%s/portio/port0/size", dirname); - if (parse_sysfs_value(filename, &size) < 0) { - PMD_INIT_LOG(ERR, "%s(): cannot parse size", - __func__); - return -1; - } - - /* get portio start */ - snprintf(filename, sizeof(filename), - "%s/portio/port0/start", dirname); - if (parse_sysfs_value(filename, &start) < 0) { - PMD_INIT_LOG(ERR, "%s(): cannot parse portio start", - __func__); - return -1; - } - pci_dev->mem_resource[0].addr = (void *)(uintptr_t)start; - pci_dev->mem_resource[0].len = (uint64_t)size; - PMD_INIT_LOG(DEBUG, - "PCI Port IO found start=0x%lx with size=0x%lx", - start, size); - - /* save fd */ - memset(dirname, 0, sizeof(dirname)); - snprintf(dirname, sizeof(dirname), "/dev/uio%u", uio_num); - pci_dev->intr_handle.fd = open(dirname, O_RDWR); - if (pci_dev->intr_handle.fd < 0) { - PMD_INIT_LOG(ERR, "Cannot open %s: %s\n", - dirname, strerror(errno)); - return -1; - } - - pci_dev->intr_handle.type = RTE_INTR_HANDLE_UIO; - pci_dev->driver->drv_flags |= RTE_PCI_DRV_INTR_LSC; - - return 0; -} - -/* Extract port I/O numbers from proc/ioports */ -static int virtio_resource_init_by_ioports(struct rte_pci_device *pci_dev) -{ - uint16_t start, end; - int size; - FILE *fp; - char *line = NULL; - char pci_id[16]; - int found = 0; - size_t linesz; - - snprintf(pci_id, sizeof(pci_id), PCI_PRI_FMT, - pci_dev->addr.domain, - pci_dev->addr.bus, - pci_dev->addr.devid, - pci_dev->addr.function); - - fp = fopen("/proc/ioports", "r"); - if (fp == NULL) { - PMD_INIT_LOG(ERR, "%s(): can't open ioports", __func__); - return -1; - } - - while (getdelim(&line, &linesz, '\n', fp) > 0) { - char *ptr = line; - char *left; - int n; - - n = strcspn(ptr, ":"); - ptr[n] = 0; - left = &ptr[n+1]; - - while (*left && isspace(*left)) - left++; - - if (!strncmp(left, pci_id, strlen(pci_id))) { - found = 1; - - while (*ptr && isspace(*ptr)) - ptr++; - - sscanf(ptr, "%04hx-%04hx", &start, &end); - size = end - start + 1; - - break; + vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); + if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) { + PMD_INIT_LOG(ERR, + "failed to set FEATURES_OK status!"); + return -1; } } - free(line); - fclose(fp); - - if (!found) - return -1; - - pci_dev->mem_resource[0].addr = (void *)(uintptr_t)(uint32_t)start; - pci_dev->mem_resource[0].len = (uint64_t)size; - PMD_INIT_LOG(DEBUG, - "PCI Port IO found start=0x%x with size=0x%x", - start, size); - - /* can't support lsc interrupt without uio */ - pci_dev->driver->drv_flags &= ~RTE_PCI_DRV_INTR_LSC; - - return 0; -} - -/* Extract I/O port numbers from sysfs */ -static int virtio_resource_init(struct rte_pci_device *pci_dev) -{ - if (virtio_resource_init_by_uio(pci_dev) == 0) - return 0; - else - return virtio_resource_init_by_ioports(pci_dev); -} - -#else -static int -virtio_has_msix(const struct rte_pci_addr *loc __rte_unused) -{ - /* nic_uio does not enable interrupts, return 0 (false). */ - return 0; -} - -static int virtio_resource_init(struct rte_pci_device *pci_dev __rte_unused) -{ - /* no setup required */ return 0; } -#endif /* * Process Virtio Config changed interrupt and call the callback @@ -1111,8 +1033,8 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) struct virtio_hw *hw = eth_dev->data->dev_private; struct virtio_net_config *config; struct virtio_net_config local_config; - uint32_t offset_conf = sizeof(config->mac); struct rte_pci_device *pci_dev; + int ret; RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr)); @@ -1125,20 +1047,19 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) } /* Allocate memory for storing MAC addresses */ - eth_dev->data->mac_addrs = rte_zmalloc("virtio", ETHER_ADDR_LEN, 0); + eth_dev->data->mac_addrs = rte_zmalloc("virtio", VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN, 0); if (eth_dev->data->mac_addrs == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to store MAC addresses", - ETHER_ADDR_LEN); + VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN); return -ENOMEM; } pci_dev = eth_dev->pci_dev; - if (virtio_resource_init(pci_dev) < 0) - return -1; - hw->use_msix = virtio_has_msix(&pci_dev->addr); - hw->io_base = (uint32_t)(uintptr_t)pci_dev->mem_resource[0].addr; + ret = vtpci_init(pci_dev, hw); + if (ret) + return ret; /* Reset the device although not necessary at startup */ vtpci_reset(hw); @@ -1148,12 +1069,20 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) /* Tell the host we've known how to drive the device. */ vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); - virtio_negotiate_features(hw); + if (virtio_negotiate_features(hw) < 0) + return -1; + + /* If host does not support status then disable LSC */ + if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) + pci_dev->driver->drv_flags &= ~RTE_PCI_DRV_INTR_LSC; + + rte_eth_copy_pci_info(eth_dev, pci_dev); rx_func_get(eth_dev); /* Setting up rx_header size for the device */ - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) + if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || + vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); else hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr); @@ -1170,8 +1099,14 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { config = &local_config; + vtpci_read_dev_config(hw, + offsetof(struct virtio_net_config, mac), + &config->mac, sizeof(config->mac)); + if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { - offset_conf += sizeof(config->status); + vtpci_read_dev_config(hw, + offsetof(struct virtio_net_config, status), + &config->status, sizeof(config->status)); } else { PMD_INIT_LOG(DEBUG, "VIRTIO_NET_F_STATUS is not supported"); @@ -1179,15 +1114,16 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) } if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) { - offset_conf += sizeof(config->max_virtqueue_pairs); + vtpci_read_dev_config(hw, + offsetof(struct virtio_net_config, max_virtqueue_pairs), + &config->max_virtqueue_pairs, + sizeof(config->max_virtqueue_pairs)); } else { PMD_INIT_LOG(DEBUG, "VIRTIO_NET_F_MQ is not supported"); config->max_virtqueue_pairs = 1; } - vtpci_read_dev_config(hw, 0, (uint8_t *)config, offset_conf); - hw->max_rx_queues = (VIRTIO_MAX_RX_QUEUES < config->max_virtqueue_pairs) ? VIRTIO_MAX_RX_QUEUES : config->max_virtqueue_pairs; @@ -1212,9 +1148,6 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) hw->max_tx_queues = 1; } - eth_dev->data->nb_rx_queues = hw->max_rx_queues; - eth_dev->data->nb_tx_queues = hw->max_tx_queues; - PMD_INIT_LOG(DEBUG, "hw->max_rx_queues=%d hw->max_tx_queues=%d", hw->max_rx_queues, hw->max_tx_queues); PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", @@ -1231,12 +1164,51 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) return 0; } +static int +eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev; + struct virtio_hw *hw = eth_dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) + return -EPERM; + + /* Close it anyway since there's no way to know if closed */ + virtio_dev_close(eth_dev); + + pci_dev = eth_dev->pci_dev; + + eth_dev->dev_ops = NULL; + eth_dev->tx_pkt_burst = NULL; + eth_dev->rx_pkt_burst = NULL; + + virtio_dev_queue_release(hw->cvq); + + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + + /* reset interrupt callback */ + if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) + rte_intr_callback_unregister(&pci_dev->intr_handle, + virtio_interrupt_handler, + eth_dev); + rte_eal_pci_unmap_device(pci_dev); + + PMD_INIT_LOG(DEBUG, "dev_uninit completed"); + + return 0; +} + static struct eth_driver rte_virtio_pmd = { .pci_drv = { .name = "rte_virtio_pmd", .id_table = pci_id_virtio_map, + .drv_flags = RTE_PCI_DRV_DETACHABLE, }, .eth_dev_init = eth_virtio_dev_init, + .eth_dev_uninit = eth_virtio_dev_uninit, .dev_private_size = sizeof(struct virtio_hw), }; @@ -1259,19 +1231,6 @@ rte_virtio_pmd_init(const char *name __rte_unused, return 0; } -/* - * Only 1 queue is supported, no queue release related operation - */ -static void -virtio_dev_rx_queue_release(__rte_unused void *rxq) -{ -} - -static void -virtio_dev_tx_queue_release(__rte_unused void *txq) -{ -} - /* * Configure virtio device * It returns 0 on success. @@ -1287,7 +1246,7 @@ virtio_dev_configure(struct rte_eth_dev *dev) if (rxmode->hw_ip_checksum) { PMD_DRV_LOG(ERR, "HW IP checksum not supported"); - return (-EINVAL); + return -EINVAL; } hw->vlan_strip = rxmode->hw_vlan_strip; @@ -1317,9 +1276,8 @@ virtio_dev_start(struct rte_eth_dev *dev) struct rte_pci_device *pci_dev = dev->pci_dev; /* check if lsc interrupt feature is enabled */ - if ((dev->data->dev_conf.intr_conf.lsc) && - (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC)) { - if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { + if (dev->data->dev_conf.intr_conf.lsc) { + if (!(pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC)) { PMD_DRV_LOG(ERR, "link status not supported by host"); return -ENOTSUP; } @@ -1379,6 +1337,8 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) "Before freeing rxq[%d] used and unused buf", i); VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); + PMD_INIT_LOG(DEBUG, "rx_queues[%d]=%p", + i, dev->data->rx_queues[i]); while ((buf = (struct rte_mbuf *)virtqueue_detatch_unused( dev->data->rx_queues[i])) != NULL) { rte_pktmbuf_free(buf); @@ -1419,9 +1379,12 @@ static void virtio_dev_stop(struct rte_eth_dev *dev) { struct rte_eth_link link; + struct virtio_hw *hw = dev->data->dev_private; PMD_INIT_LOG(DEBUG, "stop"); + hw->started = 0; + if (dev->data->dev_conf.intr_conf.lsc) rte_intr_disable(&dev->pci_dev->intr_handle); @@ -1447,16 +1410,16 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet offsetof(struct virtio_net_config, status), &status, sizeof(status)); if ((status & VIRTIO_NET_S_LINK_UP) == 0) { - link.link_status = 0; + link.link_status = ETH_LINK_DOWN; PMD_INIT_LOG(DEBUG, "Port %d is down", dev->data->port_id); } else { - link.link_status = 1; + link.link_status = ETH_LINK_UP; PMD_INIT_LOG(DEBUG, "Port %d is up", dev->data->port_id); } } else { - link.link_status = 1; /* Link up */ + link.link_status = ETH_LINK_UP; } virtio_dev_atomic_write_link_status(dev, &link);