X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_ethdev.c;h=090d94615e9c45fef57db3f0f108a712ab2e1ed5;hb=da573c0e4205d818cd602eaa27c720896f3b6f1c;hp=b540f45821afff0f5e9120d29ad1f6af28ae3a1b;hpb=c4fa9cc7c52131a0d537a11dc676259af0f1a7f3;p=dpdk.git diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index b540f45821..090d94615e 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -9,14 +9,11 @@ #include #include -#include #include #include #include #include #include -#include -#include #include #include #include @@ -24,15 +21,15 @@ #include #include #include - #include +#include #include #include #include #include #include "virtio_ethdev.h" -#include "virtio_pci.h" +#include "virtio.h" #include "virtio_logs.h" #include "virtqueue.h" #include "virtio_rxtx.h" @@ -136,7 +133,7 @@ virtio_send_command_packed(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, int *dlen, int pkt_num) { - struct virtqueue *vq = cvq->vq; + struct virtqueue *vq = virtnet_cq_to_vq(cvq); int head; struct vring_packed_desc *desc = vq->vq_packed.ring.desc; struct virtio_pmd_ctrl *result; @@ -232,7 +229,7 @@ virtio_send_command_split(struct virtnet_ctl *cvq, int *dlen, int pkt_num) { struct virtio_pmd_ctrl *result; - struct virtqueue *vq = cvq->vq; + struct virtqueue *vq = virtnet_cq_to_vq(cvq); uint32_t head, i; int k, sum = 0; @@ -319,13 +316,13 @@ virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, ctrl->status = status; - if (!cvq || !cvq->vq) { + if (!cvq) { PMD_INIT_LOG(ERR, "Control queue is not supported."); return -1; } rte_spinlock_lock(&cvq->lock); - vq = cvq->vq; + vq = virtnet_cq_to_vq(cvq); PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, " "vq->hw->cvq = %p vq = %p", @@ -339,7 +336,7 @@ virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, memcpy(cvq->virtio_net_hdr_mz->addr, ctrl, sizeof(struct virtio_pmd_ctrl)); - if (vtpci_packed_queue(vq->hw)) + if (virtio_with_packed_queue(vq->hw)) result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num); else result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num); @@ -383,7 +380,7 @@ virtio_get_nr_vq(struct virtio_hw *hw) { uint16_t nr_vq = hw->max_queue_pairs * 2; - if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) + if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) nr_vq += 1; return nr_vq; @@ -405,14 +402,14 @@ virtio_init_vring(struct virtqueue *vq) vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); vq->vq_free_cnt = vq->vq_nentries; memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); - if (vtpci_packed_queue(vq->hw)) { + if (virtio_with_packed_queue(vq->hw)) { vring_init_packed(&vq->vq_packed.ring, ring_mem, - VIRTIO_PCI_VRING_ALIGN, size); + VIRTIO_VRING_ALIGN, size); vring_desc_init_packed(vq, size); } else { struct vring *vr = &vq->vq_split.ring; - vring_init_split(vr, ring_mem, VIRTIO_PCI_VRING_ALIGN, size); + vring_init_split(vr, ring_mem, VIRTIO_VRING_ALIGN, size); vring_desc_init_split(vr->desc, size); } /* @@ -422,7 +419,7 @@ virtio_init_vring(struct virtqueue *vq) } static int -virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) +virtio_init_queue(struct rte_eth_dev *dev, uint16_t queue_idx) { char vq_name[VIRTQUEUE_MAX_NAME_SZ]; char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ]; @@ -435,31 +432,32 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) struct virtqueue *vq; size_t sz_hdr_mz = 0; void *sw_ring = NULL; - int queue_type = virtio_get_queue_type(hw, vtpci_queue_idx); + int queue_type = virtio_get_queue_type(hw, queue_idx); int ret; int numa_node = dev->device->numa_node; + struct rte_mbuf *fake_mbuf = NULL; PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d", - vtpci_queue_idx, numa_node); + queue_idx, numa_node); /* * Read the virtqueue size from the Queue Size field * Always power of 2 and if 0 virtqueue does not exist */ - vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx); + vq_size = VIRTIO_OPS(hw)->get_queue_num(hw, queue_idx); PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size); if (vq_size == 0) { PMD_INIT_LOG(ERR, "virtqueue does not exist"); return -EINVAL; } - if (!vtpci_packed_queue(hw) && !rte_is_power_of_2(vq_size)) { + if (!virtio_with_packed_queue(hw) && !rte_is_power_of_2(vq_size)) { PMD_INIT_LOG(ERR, "split virtqueue size is not power of 2"); return -EINVAL; } snprintf(vq_name, sizeof(vq_name), "port%d_vq%d", - dev->data->port_id, vtpci_queue_idx); + dev->data->port_id, queue_idx); size = RTE_ALIGN_CEIL(sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra), @@ -472,7 +470,7 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region); } else if (queue_type == VTNET_CQ) { /* Allocate a page for control vq command, data and status */ - sz_hdr_mz = PAGE_SIZE; + sz_hdr_mz = rte_mem_page_size(); } vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, @@ -481,12 +479,12 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) PMD_INIT_LOG(ERR, "can not allocate vq"); return -ENOMEM; } - hw->vqs[vtpci_queue_idx] = vq; + hw->vqs[queue_idx] = vq; vq->hw = hw; - vq->vq_queue_index = vtpci_queue_idx; + vq->vq_queue_index = queue_idx; vq->vq_nentries = vq_size; - if (vtpci_packed_queue(hw)) { + if (virtio_with_packed_queue(hw)) { vq->vq_packed.used_wrap_counter = 1; vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL; vq->vq_packed.event_flags_shadow = 0; @@ -497,20 +495,20 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) /* * Reserve a memzone for vring elements */ - size = vring_size(hw, vq_size, VIRTIO_PCI_VRING_ALIGN); - vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); + size = vring_size(hw, vq_size, VIRTIO_VRING_ALIGN); + vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN); PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size, vq->vq_ring_size); mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, numa_node, RTE_MEMZONE_IOVA_CONTIG, - VIRTIO_PCI_VRING_ALIGN); + VIRTIO_VRING_ALIGN); if (mz == NULL) { if (rte_errno == EEXIST) mz = rte_memzone_lookup(vq_name); if (mz == NULL) { ret = -ENOMEM; - goto fail_q_alloc; + goto free_vq; } } @@ -527,7 +525,7 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) if (sz_hdr_mz) { snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr", - dev->data->port_id, vtpci_queue_idx); + dev->data->port_id, queue_idx); hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz, numa_node, RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE); @@ -536,7 +534,7 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) hdr_mz = rte_memzone_lookup(vq_hdr_name); if (hdr_mz == NULL) { ret = -ENOMEM; - goto fail_q_alloc; + goto free_mz; } } } @@ -550,28 +548,34 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) if (!sw_ring) { PMD_INIT_LOG(ERR, "can not allocate RX soft ring"); ret = -ENOMEM; - goto fail_q_alloc; + goto free_hdr_mz; + } + + fake_mbuf = rte_zmalloc_socket("sw_ring", sizeof(*fake_mbuf), + RTE_CACHE_LINE_SIZE, numa_node); + if (!fake_mbuf) { + PMD_INIT_LOG(ERR, "can not allocate fake mbuf"); + ret = -ENOMEM; + goto free_sw_ring; } vq->sw_ring = sw_ring; rxvq = &vq->rxq; - rxvq->vq = vq; rxvq->port_id = dev->data->port_id; rxvq->mz = mz; + rxvq->fake_mbuf = fake_mbuf; } else if (queue_type == VTNET_TQ) { txvq = &vq->txq; - txvq->vq = vq; txvq->port_id = dev->data->port_id; txvq->mz = mz; txvq->virtio_net_hdr_mz = hdr_mz; txvq->virtio_net_hdr_mem = hdr_mz->iova; } else if (queue_type == VTNET_CQ) { cvq = &vq->cq; - cvq->vq = vq; cvq->mz = mz; cvq->virtio_net_hdr_mz = hdr_mz; cvq->virtio_net_hdr_mem = hdr_mz->iova; - memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); + memset(cvq->virtio_net_hdr_mz->addr, 0, rte_mem_page_size()); hw->cvq = cvq; } @@ -584,7 +588,7 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) memset(txr, 0, vq_size * sizeof(*txr)); for (i = 0; i < vq_size; i++) { /* first indirect descriptor is always the tx header */ - if (!vtpci_packed_queue(hw)) { + if (!virtio_with_packed_queue(hw)) { struct vring_desc *start_dp = txr[i].tx_indir; vring_desc_init_split(start_dp, RTE_DIM(txr[i].tx_indir)); @@ -608,17 +612,24 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) } } - if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) { + if (VIRTIO_OPS(hw)->setup_queue(hw, vq) < 0) { PMD_INIT_LOG(ERR, "setup_queue failed"); - return -EINVAL; + ret = -EINVAL; + goto clean_vq; } return 0; -fail_q_alloc: +clean_vq: + hw->cvq = NULL; + rte_free(fake_mbuf); +free_sw_ring: rte_free(sw_ring); +free_hdr_mz: rte_memzone_free(hdr_mz); +free_mz: rte_memzone_free(mz); +free_vq: rte_free(vq); return ret; @@ -642,6 +653,7 @@ virtio_free_queues(struct virtio_hw *hw) queue_type = virtio_get_queue_type(hw, i); if (queue_type == VTNET_RQ) { + rte_free(vq->rxq.fake_mbuf); rte_free(vq->sw_ring); rte_memzone_free(vq->rxq.mz); } else if (queue_type == VTNET_TQ) { @@ -699,11 +711,11 @@ virtio_dev_close(struct rte_eth_dev *dev) if (!hw->opened) return 0; - hw->opened = false; + hw->opened = 0; /* reset the NIC */ if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) - VTPCI_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR); + VIRTIO_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR); if (intr_conf->rxq) virtio_queues_unbind_intr(dev); @@ -714,11 +726,11 @@ virtio_dev_close(struct rte_eth_dev *dev) dev->intr_handle->intr_vec = NULL; } - vtpci_reset(hw); + virtio_reset(hw); virtio_dev_free_mbufs(dev); virtio_free_queues(hw); - return VTPCI_OPS(hw)->dev_close(hw); + return VIRTIO_OPS(hw)->dev_close(hw); } static int @@ -729,7 +741,7 @@ virtio_dev_promiscuous_enable(struct rte_eth_dev *dev) int dlen[1]; int ret; - if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { PMD_INIT_LOG(INFO, "host does not support rx control"); return -ENOTSUP; } @@ -756,7 +768,7 @@ virtio_dev_promiscuous_disable(struct rte_eth_dev *dev) int dlen[1]; int ret; - if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { PMD_INIT_LOG(INFO, "host does not support rx control"); return -ENOTSUP; } @@ -783,7 +795,7 @@ virtio_dev_allmulticast_enable(struct rte_eth_dev *dev) int dlen[1]; int ret; - if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { PMD_INIT_LOG(INFO, "host does not support rx control"); return -ENOTSUP; } @@ -810,7 +822,7 @@ virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) int dlen[1]; int ret; - if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { + if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { PMD_INIT_LOG(INFO, "host does not support rx control"); return -ENOTSUP; } @@ -854,7 +866,7 @@ virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { struct virtio_hw *hw = dev->data->dev_private; struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; - struct virtqueue *vq = rxvq->vq; + struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); virtqueue_enable_intr(vq); virtio_mb(hw->weak_barriers); @@ -865,7 +877,7 @@ static int virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; - struct virtqueue *vq = rxvq->vq; + struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); virtqueue_disable_intr(vq); return 0; @@ -1096,7 +1108,7 @@ virtio_dev_stats_reset(struct rte_eth_dev *dev) static void virtio_set_hwaddr(struct virtio_hw *hw) { - vtpci_write_dev_config(hw, + virtio_write_dev_config(hw, offsetof(struct virtio_net_config, mac), &hw->mac_addr, RTE_ETHER_ADDR_LEN); } @@ -1104,8 +1116,8 @@ virtio_set_hwaddr(struct virtio_hw *hw) static void virtio_get_hwaddr(struct virtio_hw *hw) { - if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) { - vtpci_read_dev_config(hw, + if (virtio_with_feature(hw, VIRTIO_NET_F_MAC)) { + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, mac), &hw->mac_addr, RTE_ETHER_ADDR_LEN); } else { @@ -1122,7 +1134,7 @@ virtio_mac_table_set(struct virtio_hw *hw, struct virtio_pmd_ctrl ctrl; int err, len[2]; - if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { + if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { PMD_DRV_LOG(INFO, "host does not support mac table"); return -1; } @@ -1217,7 +1229,7 @@ virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN); /* Use atomic update if available */ - if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { + if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { struct virtio_pmd_ctrl ctrl; int len = RTE_ETHER_ADDR_LEN; @@ -1228,7 +1240,7 @@ virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) return virtio_send_command(hw->cvq, &ctrl, &len, 1); } - if (!vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) + if (!virtio_with_feature(hw, VIRTIO_NET_F_MAC)) return -ENOTSUP; virtio_set_hwaddr(hw); @@ -1242,7 +1254,7 @@ virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) struct virtio_pmd_ctrl ctrl; int len; - if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) + if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) return -ENOTSUP; ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN; @@ -1261,8 +1273,8 @@ virtio_intr_unmask(struct rte_eth_dev *dev) if (rte_intr_ack(dev->intr_handle) < 0) return -1; - if (VTPCI_OPS(hw)->intr_detect) - VTPCI_OPS(hw)->intr_detect(hw); + if (VIRTIO_OPS(hw)->intr_detect) + VIRTIO_OPS(hw)->intr_detect(hw); return 0; } @@ -1275,8 +1287,8 @@ virtio_intr_enable(struct rte_eth_dev *dev) if (rte_intr_enable(dev->intr_handle) < 0) return -1; - if (VTPCI_OPS(hw)->intr_detect) - VTPCI_OPS(hw)->intr_detect(hw); + if (VIRTIO_OPS(hw)->intr_detect) + VIRTIO_OPS(hw)->intr_detect(hw); return 0; } @@ -1289,14 +1301,14 @@ virtio_intr_disable(struct rte_eth_dev *dev) if (rte_intr_disable(dev->intr_handle) < 0) return -1; - if (VTPCI_OPS(hw)->intr_detect) - VTPCI_OPS(hw)->intr_detect(hw); + if (VIRTIO_OPS(hw)->intr_detect) + VIRTIO_OPS(hw)->intr_detect(hw); return 0; } static int -virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features) +virtio_ethdev_negotiate_features(struct virtio_hw *hw, uint64_t req_features) { uint64_t host_features; @@ -1305,7 +1317,7 @@ virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features) req_features); /* Read device(host) feature bits */ - host_features = VTPCI_OPS(hw)->get_features(hw); + host_features = VIRTIO_OPS(hw)->get_features(hw); PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64, host_features); @@ -1313,7 +1325,7 @@ virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features) if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) { struct virtio_net_config config; - vtpci_read_dev_config(hw, + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, mtu), &config.mtu, sizeof(config.mtu)); @@ -1326,17 +1338,17 @@ virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features) * guest feature bits. */ hw->guest_features = req_features; - hw->guest_features = vtpci_negotiate_features(hw, host_features); + hw->guest_features = virtio_negotiate_features(hw, host_features); PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64, hw->guest_features); - if (VTPCI_OPS(hw)->features_ok(hw) < 0) + if (VIRTIO_OPS(hw)->features_ok(hw) < 0) return -1; - if (vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { - vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); + if (virtio_with_feature(hw, VIRTIO_F_VERSION_1)) { + virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); - if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) { + if (!(virtio_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) { PMD_INIT_LOG(ERR, "Failed to set FEATURES_OK status!"); return -1; } @@ -1455,20 +1467,20 @@ virtio_interrupt_handler(void *param) uint16_t status; /* Read interrupt status which clears interrupt */ - isr = vtpci_isr(hw); + isr = virtio_get_isr(hw); PMD_DRV_LOG(INFO, "interrupt status = %#x", isr); if (virtio_intr_unmask(dev) < 0) PMD_DRV_LOG(ERR, "interrupt enable failed"); - if (isr & VIRTIO_PCI_ISR_CONFIG) { + if (isr & VIRTIO_ISR_CONFIG) { if (virtio_dev_link_update(dev, 0) == 0) rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); - if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { - vtpci_read_dev_config(hw, + if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) { + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, status), &status, sizeof(status)); if (status & VIRTIO_NET_S_ANNOUNCE) { @@ -1487,7 +1499,7 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev) struct virtio_hw *hw = eth_dev->data->dev_private; eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare; - if (vtpci_packed_queue(hw)) { + if (virtio_with_packed_queue(hw)) { PMD_INIT_LOG(INFO, "virtio: using packed ring %s Tx path on port %u", hw->use_vec_tx ? "vectorized" : "standard", @@ -1508,14 +1520,14 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev) } } - if (vtpci_packed_queue(hw)) { + if (virtio_with_packed_queue(hw)) { if (hw->use_vec_rx) { PMD_INIT_LOG(INFO, "virtio: using packed ring vectorized Rx path on port %u", eth_dev->data->port_id); eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed_vec; - } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { PMD_INIT_LOG(INFO, "virtio: using packed ring mergeable buffer Rx path on port %u", eth_dev->data->port_id); @@ -1537,7 +1549,7 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev) "virtio: using inorder Rx path on port %u", eth_dev->data->port_id); eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder; - } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { PMD_INIT_LOG(INFO, "virtio: using mergeable buffer Rx path on port %u", eth_dev->data->port_id); @@ -1564,7 +1576,7 @@ virtio_queues_bind_intr(struct rte_eth_dev *dev) PMD_INIT_LOG(INFO, "queue/interrupt binding"); for (i = 0; i < dev->data->nb_rx_queues; ++i) { dev->intr_handle->intr_vec[i] = i + 1; - if (VTPCI_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) == + if (VIRTIO_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) == VIRTIO_MSI_NO_VECTOR) { PMD_DRV_LOG(ERR, "failed to set queue vector"); return -EBUSY; @@ -1582,7 +1594,7 @@ virtio_queues_unbind_intr(struct rte_eth_dev *dev) PMD_INIT_LOG(INFO, "queue/interrupt unbinding"); for (i = 0; i < dev->data->nb_rx_queues; ++i) - VTPCI_OPS(hw)->set_queue_irq(hw, + VIRTIO_OPS(hw)->set_queue_irq(hw, hw->vqs[i * VTNET_CQ], VIRTIO_MSI_NO_VECTOR); } @@ -1650,7 +1662,7 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) int ret; /* Reset the device although not necessary at startup */ - vtpci_reset(hw); + virtio_reset(hw); if (hw->vqs) { virtio_dev_free_mbufs(eth_dev); @@ -1658,18 +1670,17 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) } /* Tell the host we've noticed this device. */ - vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); + virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); /* Tell the host we've known how to drive the device. */ - vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); - if (virtio_negotiate_features(hw, req_features) < 0) + virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); + if (virtio_ethdev_negotiate_features(hw, req_features) < 0) return -1; - hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM); + hw->weak_barriers = !virtio_with_feature(hw, VIRTIO_F_ORDER_PLATFORM); /* If host does not support both status and MSI-X then disable LSC */ - if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS) && - hw->use_msix != VIRTIO_MSIX_NONE) + if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS) && hw->intr_lsc) eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; else eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC; @@ -1677,9 +1688,9 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; /* Setting up rx_header size for the device */ - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || - vtpci_with_feature(hw, VIRTIO_F_VERSION_1) || - vtpci_with_feature(hw, VIRTIO_F_RING_PACKED)) + if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || + virtio_with_feature(hw, VIRTIO_F_VERSION_1) || + virtio_with_packed_queue(hw)) hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); else hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr); @@ -1694,12 +1705,12 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); if (hw->speed == ETH_SPEED_NUM_UNKNOWN) { - if (vtpci_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) { + if (virtio_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) { config = &local_config; - vtpci_read_dev_config(hw, + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, speed), &config->speed, sizeof(config->speed)); - vtpci_read_dev_config(hw, + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, duplex), &config->duplex, sizeof(config->duplex)); hw->speed = config->speed; @@ -1710,15 +1721,15 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) hw->duplex = ETH_LINK_FULL_DUPLEX; PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d", hw->speed, hw->duplex); - if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { + if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { config = &local_config; - vtpci_read_dev_config(hw, + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, mac), &config->mac, sizeof(config->mac)); - if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { - vtpci_read_dev_config(hw, + if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) { + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, status), &config->status, sizeof(config->status)); } else { @@ -1727,8 +1738,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) config->status = 0; } - if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) { - vtpci_read_dev_config(hw, + if (virtio_with_feature(hw, VIRTIO_NET_F_MQ)) { + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, max_virtqueue_pairs), &config->max_virtqueue_pairs, sizeof(config->max_virtqueue_pairs)); @@ -1740,8 +1751,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) hw->max_queue_pairs = config->max_virtqueue_pairs; - if (vtpci_with_feature(hw, VIRTIO_NET_F_MTU)) { - vtpci_read_dev_config(hw, + if (virtio_with_feature(hw, VIRTIO_NET_F_MTU)) { + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, mtu), &config->mtu, sizeof(config->mtu)); @@ -1793,7 +1804,7 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) } } - vtpci_reinit_complete(hw); + virtio_reinit_complete(hw); return 0; } @@ -1842,7 +1853,6 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) return -ENOMEM; } - hw->port_id = eth_dev->data->port_id; rte_spinlock_init(&hw->state_lock); /* reset device and negotiate default features */ @@ -1851,7 +1861,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) goto err_virtio_init; if (vectorized) { - if (!vtpci_packed_queue(hw)) { + if (!virtio_with_packed_queue(hw)) { hw->use_vec_rx = 1; } else { #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM) @@ -1864,7 +1874,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) } } - hw->opened = true; + hw->opened = 1; return 0; @@ -1973,20 +1983,20 @@ exit: return ret; } -static bool +static uint8_t rx_offload_enabled(struct virtio_hw *hw) { - return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) || - vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || - vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6); + return virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) || + virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || + virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6); } -static bool +static uint8_t tx_offload_enabled(struct virtio_hw *hw) { - return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) || - vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) || - vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6); + return virtio_with_feature(hw, VIRTIO_NET_F_CSUM) || + virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) || + virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO6); } /* @@ -2059,29 +2069,29 @@ virtio_dev_configure(struct rte_eth_dev *dev) if ((rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM)) && - !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) { + !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) { PMD_DRV_LOG(ERR, "rx checksum not available on this host"); return -ENOTSUP; } if ((rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) && - (!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || - !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) { + (!virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || + !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) { PMD_DRV_LOG(ERR, "Large Receive Offload not available on this host"); return -ENOTSUP; } /* start control queue */ - if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) + if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) virtio_dev_cq_start(dev); if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP) hw->vlan_strip = 1; - if ((rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER) - && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { + if ((rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER) && + !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { PMD_DRV_LOG(ERR, "vlan filtering not available on this host"); return -ENOTSUP; @@ -2092,18 +2102,18 @@ virtio_dev_configure(struct rte_eth_dev *dev) if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) /* Enable vector (0) for Link State Intrerrupt */ - if (VTPCI_OPS(hw)->set_config_irq(hw, 0) == + if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) == VIRTIO_MSI_NO_VECTOR) { PMD_DRV_LOG(ERR, "failed to set config vector"); return -EBUSY; } - if (vtpci_packed_queue(hw)) { + if (virtio_with_packed_queue(hw)) { #if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT) if ((hw->use_vec_rx || hw->use_vec_tx) && (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) || - !vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) || - !vtpci_with_feature(hw, VIRTIO_F_VERSION_1) || + !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) || + !virtio_with_feature(hw, VIRTIO_F_VERSION_1) || rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_512)) { PMD_DRV_LOG(INFO, "disabled packed ring vectorized path for requirements not met"); @@ -2113,8 +2123,8 @@ virtio_dev_configure(struct rte_eth_dev *dev) #elif defined(RTE_ARCH_ARM) if ((hw->use_vec_rx || hw->use_vec_tx) && (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) || - !vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) || - !vtpci_with_feature(hw, VIRTIO_F_VERSION_1) || + !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) || + !virtio_with_feature(hw, VIRTIO_F_VERSION_1) || rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)) { PMD_DRV_LOG(INFO, "disabled packed ring vectorized path for requirements not met"); @@ -2127,7 +2137,7 @@ virtio_dev_configure(struct rte_eth_dev *dev) #endif if (hw->use_vec_rx) { - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { PMD_DRV_LOG(INFO, "disabled packed ring vectorized rx for mrg_rxbuf enabled"); hw->use_vec_rx = 0; @@ -2140,7 +2150,7 @@ virtio_dev_configure(struct rte_eth_dev *dev) } } } else { - if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) { + if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) { hw->use_inorder_tx = 1; hw->use_inorder_rx = 1; hw->use_vec_rx = 0; @@ -2154,7 +2164,7 @@ virtio_dev_configure(struct rte_eth_dev *dev) hw->use_vec_rx = 0; } #endif - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { PMD_DRV_LOG(INFO, "disabled split ring vectorized rx for mrg_rxbuf enabled"); hw->use_vec_rx = 0; @@ -2185,8 +2195,7 @@ static int virtio_dev_start(struct rte_eth_dev *dev) { uint16_t nb_queues, i; - struct virtnet_rx *rxvq; - struct virtnet_tx *txvq __rte_unused; + struct virtqueue *vq; struct virtio_hw *hw = dev->data->dev_private; int ret; @@ -2243,31 +2252,31 @@ virtio_dev_start(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues); for (i = 0; i < dev->data->nb_rx_queues; i++) { - rxvq = dev->data->rx_queues[i]; + vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]); /* Flush the old packets */ - virtqueue_rxvq_flush(rxvq->vq); - virtqueue_notify(rxvq->vq); + virtqueue_rxvq_flush(vq); + virtqueue_notify(vq); } for (i = 0; i < dev->data->nb_tx_queues; i++) { - txvq = dev->data->tx_queues[i]; - virtqueue_notify(txvq->vq); + vq = virtnet_txq_to_vq(dev->data->tx_queues[i]); + virtqueue_notify(vq); } PMD_INIT_LOG(DEBUG, "Notified backend at initialization"); for (i = 0; i < dev->data->nb_rx_queues; i++) { - rxvq = dev->data->rx_queues[i]; - VIRTQUEUE_DUMP(rxvq->vq); + vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]); + VIRTQUEUE_DUMP(vq); } for (i = 0; i < dev->data->nb_tx_queues; i++) { - txvq = dev->data->tx_queues[i]; - VIRTQUEUE_DUMP(txvq->vq); + vq = virtnet_txq_to_vq(dev->data->tx_queues[i]); + VIRTQUEUE_DUMP(vq); } set_rxtx_funcs(dev); - hw->started = true; + hw->started = 1; /* Initialize Link state */ virtio_dev_link_update(dev, 0); @@ -2336,7 +2345,7 @@ virtio_dev_stop(struct rte_eth_dev *dev) rte_spinlock_lock(&hw->state_lock); if (!hw->started) goto out_unlock; - hw->started = false; + hw->started = 0; if (intr_conf->lsc || intr_conf->rxq) { virtio_intr_disable(dev); @@ -2372,9 +2381,9 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet if (!hw->started) { link.link_status = ETH_LINK_DOWN; link.link_speed = ETH_SPEED_NUM_NONE; - } else if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { + } else if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) { PMD_INIT_LOG(DEBUG, "Get link status from hw"); - vtpci_read_dev_config(hw, + virtio_read_dev_config(hw, offsetof(struct virtio_net_config, status), &status, sizeof(status)); if ((status & VIRTIO_NET_S_LINK_UP) == 0) { @@ -2403,7 +2412,7 @@ virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) if (mask & ETH_VLAN_FILTER_MASK) { if ((offloads & DEV_RX_OFFLOAD_VLAN_FILTER) && - !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { + !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { PMD_DRV_LOG(NOTICE, "vlan filtering not available on this host"); @@ -2433,7 +2442,7 @@ virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN; dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS; - host_features = VTPCI_OPS(hw)->get_features(hw); + host_features = VIRTIO_OPS(hw)->get_features(hw); dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME; if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {