X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_rxtx.c;h=e96352cedb591b81ed78256c72453e5e0f60966a;hb=9eddbdb4b094;hp=8db36907be87fca01e1bbb5b4efcc8687306c968;hpb=fc3d66212fed67b499e80cae06106af64784bf54;p=dpdk.git diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index 8db36907be..e96352cedb 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -53,6 +53,7 @@ #include "virtio_logs.h" #include "virtio_ethdev.h" +#include "virtio_pci.h" #include "virtqueue.h" #include "virtio_rxtx.h" @@ -62,7 +63,13 @@ #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) #endif + +#define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \ + ETH_TXQ_FLAGS_NOOFFLOADS) + +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 static int use_simple_rxtx; +#endif static void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) @@ -232,7 +239,7 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) { idx = start_dp[idx].next; - start_dp[idx].addr = RTE_MBUF_DATA_DMA_ADDR(cookie); + start_dp[idx].addr = rte_mbuf_data_dma_addr(cookie); start_dp[idx].len = cookie->data_len; start_dp[idx].flags = VRING_DESC_F_NEXT; cookie = cookie->next; @@ -302,12 +309,13 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type) nbufs = 0; error = ENOSPC; +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 if (use_simple_rxtx) for (i = 0; i < vq->vq_nentries; i++) { vq->vq_ring.avail->ring[i] = i; vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE; } - +#endif memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf)); for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++) vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf; @@ -320,9 +328,11 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type) /****************************************** * Enqueue allocated buffers * *******************************************/ +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 if (use_simple_rxtx) error = virtqueue_enqueue_recv_refill_simple(vq, m); else +#endif error = virtqueue_enqueue_recv_refill(vq, m); if (error) { rte_pktmbuf_free(m); @@ -334,12 +344,8 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type) vq_update_avail_idx(vq); PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); - - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, - vq->vq_queue_index); - VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, - vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); } else if (queue_type == VTNET_TQ) { +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 if (use_simple_rxtx) { int mid_idx = vq->vq_nentries >> 1; for (i = 0; i < mid_idx; i++) { @@ -357,16 +363,7 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type) for (i = mid_idx; i < vq->vq_nentries; i++) vq->vq_ring.avail->ring[i] = i; } - - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, - vq->vq_queue_index); - VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, - vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); - } else { - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, - vq->vq_queue_index); - VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, - vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); +#endif } } @@ -433,7 +430,9 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, dev->data->rx_queues[queue_idx] = vq; +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 virtio_rxq_vec_setup(vq); +#endif return 0; } @@ -459,6 +458,10 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, const struct rte_eth_txconf *tx_conf) { uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; + +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + struct virtio_hw *hw = dev->data->dev_private; +#endif struct virtqueue *vq; uint16_t tx_free_thresh; int ret; @@ -471,6 +474,17 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, return -EINVAL; } +#ifdef RTE_MACHINE_CPUFLAG_SSSE3 + /* Use simple rx/tx func if single segment and no offloads */ + if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS && + !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + PMD_INIT_LOG(INFO, "Using simple rx/tx path"); + dev->tx_pkt_burst = virtio_xmit_pkts_simple; + dev->rx_pkt_burst = virtio_recv_pkts_vec; + use_simple_rxtx = 1; + } +#endif + ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx, nb_desc, socket_id, &vq); if (ret < 0) { @@ -519,6 +533,34 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) } } +static void +virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf) +{ + uint32_t s = mbuf->pkt_len; + struct ether_addr *ea; + + if (s == 64) { + vq->size_bins[1]++; + } else if (s > 64 && s < 1024) { + uint32_t bin; + + /* count zeros, and offset into correct bin */ + bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; + vq->size_bins[bin]++; + } else { + if (s < 64) + vq->size_bins[0]++; + else if (s < 1519) + vq->size_bins[6]++; + else if (s >= 1519) + vq->size_bins[7]++; + } + + ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *); + vq->multicast += is_multicast_ether_addr(ea); + vq->broadcast += is_broadcast_ether_addr(ea); +} + #define VIRTIO_MBUF_BURST_SZ 64 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) uint16_t @@ -532,7 +574,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; int error; uint32_t i, nb_enqueued; - const uint32_t hdr_size = sizeof(struct virtio_net_hdr); + uint32_t hdr_size; nb_used = VIRTQUEUE_NUSED(rxvq); @@ -552,6 +594,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) hw = rxvq->hw; nb_rx = 0; nb_enqueued = 0; + hdr_size = hw->vtnet_hdr_size; for (i = 0; i < num ; i++) { rxm = rcv_pkts[i]; @@ -568,6 +611,8 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->port = rxvq->port_id; rxm->data_off = RTE_PKTMBUF_HEADROOM; + rxm->ol_flags = 0; + rxm->vlan_tci = 0; rxm->nb_segs = 1; rxm->next = NULL; @@ -580,7 +625,9 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) VIRTIO_DUMP_PACKET(rxm, rxm->data_len); rx_pkts[nb_rx++] = rxm; + rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len; + virtio_update_packet_stats(rxvq, rxm); } rxvq->packets += nb_rx; @@ -632,7 +679,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, uint32_t seg_num; uint16_t extra_idx; uint32_t seg_res; - const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); + uint32_t hdr_size; nb_used = VIRTQUEUE_NUSED(rxvq); @@ -650,6 +697,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, seg_num = 0; extra_idx = 0; seg_res = 0; + hdr_size = hw->vtnet_hdr_size; while (i < nb_used) { struct virtio_net_hdr_mrg_rxbuf *header; @@ -686,6 +734,8 @@ virtio_recv_mergeable_pkts(void *rx_queue, rxm->data_off = RTE_PKTMBUF_HEADROOM; rxm->nb_segs = seg_num; rxm->next = NULL; + rxm->ol_flags = 0; + rxm->vlan_tci = 0; rxm->pkt_len = (uint32_t)(len[0] - hdr_size); rxm->data_len = (uint16_t)(len[0] - hdr_size); @@ -743,6 +793,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, rx_pkts[nb_rx]->data_len); rxvq->bytes += rx_pkts[nb_rx]->pkt_len; + virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]); nb_rx++; } @@ -782,7 +833,6 @@ uint16_t virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct virtqueue *txvq = tx_queue; - struct rte_mbuf *txm; uint16_t nb_used, nb_tx; int error; @@ -796,57 +846,49 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh)) virtio_xmit_cleanup(txvq, nb_used); - nb_tx = 0; - - while (nb_tx < nb_pkts) { + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + struct rte_mbuf *txm = tx_pkts[nb_tx]; /* Need one more descriptor for virtio header. */ - int need = tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1; + int need = txm->nb_segs - txvq->vq_free_cnt + 1; - /*Positive value indicates it need free vring descriptors */ + /* Positive value indicates it need free vring descriptors */ if (unlikely(need > 0)) { nb_used = VIRTQUEUE_NUSED(txvq); virtio_rmb(); need = RTE_MIN(need, (int)nb_used); virtio_xmit_cleanup(txvq, need); - need = (int)tx_pkts[nb_tx]->nb_segs - - txvq->vq_free_cnt + 1; - } - - /* - * Zero or negative value indicates it has enough free - * descriptors to use for transmitting. - */ - if (likely(need <= 0)) { - txm = tx_pkts[nb_tx]; - - /* Do VLAN tag insertion */ - if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) { - error = rte_vlan_insert(&txm); - if (unlikely(error)) { - rte_pktmbuf_free(txm); - ++nb_tx; - continue; - } + need = txm->nb_segs - txvq->vq_free_cnt + 1; + if (unlikely(need > 0)) { + PMD_TX_LOG(ERR, + "No free tx descriptors to transmit"); + break; } + } - /* Enqueue Packet buffers */ - error = virtqueue_enqueue_xmit(txvq, txm); + /* Do VLAN tag insertion */ + if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) { + error = rte_vlan_insert(&txm); if (unlikely(error)) { - if (error == ENOSPC) - PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0"); - else if (error == EMSGSIZE) - PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1"); - else - PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error); - break; + rte_pktmbuf_free(txm); + continue; } - nb_tx++; - txvq->bytes += txm->pkt_len; - } else { - PMD_TX_LOG(ERR, "No free tx descriptors to transmit"); + } + + /* Enqueue Packet buffers */ + error = virtqueue_enqueue_xmit(txvq, txm); + if (unlikely(error)) { + if (error == ENOSPC) + PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0"); + else if (error == EMSGSIZE) + PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1"); + else + PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error); break; } + + txvq->bytes += txm->pkt_len; + virtio_update_packet_stats(txvq, txm); } txvq->packets += nb_tx;