X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_rxtx.c;h=e96352cedb591b81ed78256c72453e5e0f60966a;hb=d6b324c00fc933f757e68c54c0e50c92826d83f0;hp=091c7fb4780f1c09df09a65854072375d8e39d66;hpb=72514b5d55431c4798ad25bcf6c67bc62b0a5800;p=dpdk.git diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index 091c7fb478..e96352cedb 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -53,7 +53,9 @@ #include "virtio_logs.h" #include "virtio_ethdev.h" +#include "virtio_pci.h" #include "virtqueue.h" +#include "virtio_rxtx.h" #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) @@ -61,6 +63,14 @@ #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) #endif + +#define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \ + ETH_TXQ_FLAGS_NOOFFLOADS) + +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 +static int use_simple_rxtx; +#endif + static void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) { @@ -206,7 +216,7 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) uint16_t seg_num = cookie->nb_segs; uint16_t needed = 1 + seg_num; uint16_t head_idx, idx; - uint16_t head_size = txvq->hw->vtnet_hdr_size; + size_t head_size = txvq->hw->vtnet_hdr_size; if (unlikely(txvq->vq_free_cnt == 0)) return -ENOSPC; @@ -224,12 +234,12 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) start_dp = txvq->vq_ring.desc; start_dp[idx].addr = txvq->virtio_net_hdr_mem + idx * head_size; - start_dp[idx].len = (uint32_t)head_size; + start_dp[idx].len = head_size; start_dp[idx].flags = VRING_DESC_F_NEXT; for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) { idx = start_dp[idx].next; - start_dp[idx].addr = RTE_MBUF_DATA_DMA_ADDR(cookie); + start_dp[idx].addr = rte_mbuf_data_dma_addr(cookie); start_dp[idx].len = cookie->data_len; start_dp[idx].flags = VRING_DESC_F_NEXT; cookie = cookie->next; @@ -298,6 +308,18 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type) /* Allocate blank mbufs for the each rx descriptor */ nbufs = 0; error = ENOSPC; + +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + if (use_simple_rxtx) + for (i = 0; i < vq->vq_nentries; i++) { + vq->vq_ring.avail->ring[i] = i; + vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE; + } +#endif + memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf)); + for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++) + vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf; + while (!virtqueue_full(vq)) { m = rte_rxmbuf_alloc(vq->mpool); if (m == NULL) @@ -306,8 +328,12 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type) /****************************************** * Enqueue allocated buffers * *******************************************/ - error = virtqueue_enqueue_recv_refill(vq, m); - +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + if (use_simple_rxtx) + error = virtqueue_enqueue_recv_refill_simple(vq, m); + else +#endif + error = virtqueue_enqueue_recv_refill(vq, m); if (error) { rte_pktmbuf_free(m); break; @@ -318,21 +344,26 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type) vq_update_avail_idx(vq); PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); - - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, - vq->vq_queue_index); - VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, - vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); } else if (queue_type == VTNET_TQ) { - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, - vq->vq_queue_index); - VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, - vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); - } else { - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, - vq->vq_queue_index); - VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, - vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + if (use_simple_rxtx) { + int mid_idx = vq->vq_nentries >> 1; + for (i = 0; i < mid_idx; i++) { + vq->vq_ring.avail->ring[i] = i + mid_idx; + vq->vq_ring.desc[i + mid_idx].next = i; + vq->vq_ring.desc[i + mid_idx].addr = + vq->virtio_net_hdr_mem + + mid_idx * vq->hw->vtnet_hdr_size; + vq->vq_ring.desc[i + mid_idx].len = + vq->hw->vtnet_hdr_size; + vq->vq_ring.desc[i + mid_idx].flags = + VRING_DESC_F_NEXT; + vq->vq_ring.desc[i].flags = 0; + } + for (i = mid_idx; i < vq->vq_nentries; i++) + vq->vq_ring.avail->ring[i] = i; + } +#endif } } @@ -390,7 +421,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx, nb_desc, socket_id, &vq); if (ret < 0) { - PMD_INIT_LOG(ERR, "tvq initialization failed"); + PMD_INIT_LOG(ERR, "rvq initialization failed"); return ret; } @@ -398,9 +429,20 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, vq->mpool = mp; dev->data->rx_queues[queue_idx] = vq; + +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + virtio_rxq_vec_setup(vq); +#endif + return 0; } +void +virtio_dev_rx_queue_release(void *rxq) +{ + virtio_dev_queue_release(rxq); +} + /* * struct rte_eth_dev *dev: Used to update dev * uint16_t nb_desc: Defaults to values read from config space @@ -416,6 +458,10 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, const struct rte_eth_txconf *tx_conf) { uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; + +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + struct virtio_hw *hw = dev->data->dev_private; +#endif struct virtqueue *vq; uint16_t tx_free_thresh; int ret; @@ -428,6 +474,17 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, return -EINVAL; } +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + /* Use simple rx/tx func if single segment and no offloads */ + if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS && + !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + PMD_INIT_LOG(INFO, "Using simple rx/tx path"); + dev->tx_pkt_burst = virtio_xmit_pkts_simple; + dev->rx_pkt_burst = virtio_recv_pkts_vec; + use_simple_rxtx = 1; + } +#endif + ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx, nb_desc, socket_id, &vq); if (ret < 0) { @@ -455,6 +512,12 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, return 0; } +void +virtio_dev_tx_queue_release(void *txq) +{ + virtio_dev_queue_release(txq); +} + static void virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) { @@ -470,6 +533,34 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) } } +static void +virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf) +{ + uint32_t s = mbuf->pkt_len; + struct ether_addr *ea; + + if (s == 64) { + vq->size_bins[1]++; + } else if (s > 64 && s < 1024) { + uint32_t bin; + + /* count zeros, and offset into correct bin */ + bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; + vq->size_bins[bin]++; + } else { + if (s < 64) + vq->size_bins[0]++; + else if (s < 1519) + vq->size_bins[6]++; + else if (s >= 1519) + vq->size_bins[7]++; + } + + ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *); + vq->multicast += is_multicast_ether_addr(ea); + vq->broadcast += is_broadcast_ether_addr(ea); +} + #define VIRTIO_MBUF_BURST_SZ 64 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) uint16_t @@ -483,7 +574,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; int error; uint32_t i, nb_enqueued; - const uint32_t hdr_size = sizeof(struct virtio_net_hdr); + uint32_t hdr_size; nb_used = VIRTQUEUE_NUSED(rxvq); @@ -503,6 +594,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) hw = rxvq->hw; nb_rx = 0; nb_enqueued = 0; + hdr_size = hw->vtnet_hdr_size; for (i = 0; i < num ; i++) { rxm = rcv_pkts[i]; @@ -519,6 +611,8 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->port = rxvq->port_id; rxm->data_off = RTE_PKTMBUF_HEADROOM; + rxm->ol_flags = 0; + rxm->vlan_tci = 0; rxm->nb_segs = 1; rxm->next = NULL; @@ -531,7 +625,9 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) VIRTIO_DUMP_PACKET(rxm, rxm->data_len); rx_pkts[nb_rx++] = rxm; + rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len; + virtio_update_packet_stats(rxvq, rxm); } rxvq->packets += nb_rx; @@ -583,7 +679,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, uint32_t seg_num; uint16_t extra_idx; uint32_t seg_res; - const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); + uint32_t hdr_size; nb_used = VIRTQUEUE_NUSED(rxvq); @@ -601,6 +697,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, seg_num = 0; extra_idx = 0; seg_res = 0; + hdr_size = hw->vtnet_hdr_size; while (i < nb_used) { struct virtio_net_hdr_mrg_rxbuf *header; @@ -637,6 +734,8 @@ virtio_recv_mergeable_pkts(void *rx_queue, rxm->data_off = RTE_PKTMBUF_HEADROOM; rxm->nb_segs = seg_num; rxm->next = NULL; + rxm->ol_flags = 0; + rxm->vlan_tci = 0; rxm->pkt_len = (uint32_t)(len[0] - hdr_size); rxm->data_len = (uint16_t)(len[0] - hdr_size); @@ -694,6 +793,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, rx_pkts[nb_rx]->data_len); rxvq->bytes += rx_pkts[nb_rx]->pkt_len; + virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]); nb_rx++; } @@ -733,7 +833,6 @@ uint16_t virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct virtqueue *txvq = tx_queue; - struct rte_mbuf *txm; uint16_t nb_used, nb_tx; int error; @@ -747,57 +846,49 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh)) virtio_xmit_cleanup(txvq, nb_used); - nb_tx = 0; - - while (nb_tx < nb_pkts) { + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + struct rte_mbuf *txm = tx_pkts[nb_tx]; /* Need one more descriptor for virtio header. */ - int need = tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1; + int need = txm->nb_segs - txvq->vq_free_cnt + 1; - /*Positive value indicates it need free vring descriptors */ + /* Positive value indicates it need free vring descriptors */ if (unlikely(need > 0)) { nb_used = VIRTQUEUE_NUSED(txvq); virtio_rmb(); need = RTE_MIN(need, (int)nb_used); virtio_xmit_cleanup(txvq, need); - need = (int)tx_pkts[nb_tx]->nb_segs - - txvq->vq_free_cnt + 1; - } - - /* - * Zero or negative value indicates it has enough free - * descriptors to use for transmitting. - */ - if (likely(need <= 0)) { - txm = tx_pkts[nb_tx]; - - /* Do VLAN tag insertion */ - if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) { - error = rte_vlan_insert(&txm); - if (unlikely(error)) { - rte_pktmbuf_free(txm); - ++nb_tx; - continue; - } + need = txm->nb_segs - txvq->vq_free_cnt + 1; + if (unlikely(need > 0)) { + PMD_TX_LOG(ERR, + "No free tx descriptors to transmit"); + break; } + } - /* Enqueue Packet buffers */ - error = virtqueue_enqueue_xmit(txvq, txm); + /* Do VLAN tag insertion */ + if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) { + error = rte_vlan_insert(&txm); if (unlikely(error)) { - if (error == ENOSPC) - PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0"); - else if (error == EMSGSIZE) - PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1"); - else - PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error); - break; + rte_pktmbuf_free(txm); + continue; } - nb_tx++; - txvq->bytes += txm->pkt_len; - } else { - PMD_TX_LOG(ERR, "No free tx descriptors to transmit"); + } + + /* Enqueue Packet buffers */ + error = virtqueue_enqueue_xmit(txvq, txm); + if (unlikely(error)) { + if (error == ENOSPC) + PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0"); + else if (error == EMSGSIZE) + PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1"); + else + PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error); break; } + + txvq->bytes += txm->pkt_len; + virtio_update_packet_stats(txvq, txm); } txvq->packets += nb_tx;