#include "virtio_logs.h"
#include "virtio_ethdev.h"
+#include "virtio_pci.h"
#include "virtqueue.h"
+#include "virtio_rxtx.h"
#ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
#define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
#define VIRTIO_DUMP_PACKET(m, len) do { } while (0)
#endif
+
+#define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
+ ETH_TXQ_FLAGS_NOOFFLOADS)
+
+#ifdef RTE_MACHINE_CPUFLAG_SSSE3
+static int use_simple_rxtx;
+#endif
+
static void
vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
{
uint16_t seg_num = cookie->nb_segs;
uint16_t needed = 1 + seg_num;
uint16_t head_idx, idx;
- uint16_t head_size = txvq->hw->vtnet_hdr_size;
+ size_t head_size = txvq->hw->vtnet_hdr_size;
if (unlikely(txvq->vq_free_cnt == 0))
return -ENOSPC;
start_dp = txvq->vq_ring.desc;
start_dp[idx].addr =
txvq->virtio_net_hdr_mem + idx * head_size;
- start_dp[idx].len = (uint32_t)head_size;
+ start_dp[idx].len = head_size;
start_dp[idx].flags = VRING_DESC_F_NEXT;
for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) {
idx = start_dp[idx].next;
- start_dp[idx].addr = RTE_MBUF_DATA_DMA_ADDR(cookie);
+ start_dp[idx].addr = rte_mbuf_data_dma_addr(cookie);
start_dp[idx].len = cookie->data_len;
start_dp[idx].flags = VRING_DESC_F_NEXT;
cookie = cookie->next;
/* Allocate blank mbufs for the each rx descriptor */
nbufs = 0;
error = ENOSPC;
+
+#ifdef RTE_MACHINE_CPUFLAG_SSSE3
+ if (use_simple_rxtx)
+ for (i = 0; i < vq->vq_nentries; i++) {
+ vq->vq_ring.avail->ring[i] = i;
+ vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
+ }
+#endif
+ memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf));
+ for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++)
+ vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf;
+
while (!virtqueue_full(vq)) {
m = rte_rxmbuf_alloc(vq->mpool);
if (m == NULL)
/******************************************
* Enqueue allocated buffers *
*******************************************/
- error = virtqueue_enqueue_recv_refill(vq, m);
-
+#ifdef RTE_MACHINE_CPUFLAG_SSSE3
+ if (use_simple_rxtx)
+ error = virtqueue_enqueue_recv_refill_simple(vq, m);
+ else
+#endif
+ error = virtqueue_enqueue_recv_refill(vq, m);
if (error) {
rte_pktmbuf_free(m);
break;
vq_update_avail_idx(vq);
PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
-
- VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
- vq->vq_queue_index);
- VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
- vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
} else if (queue_type == VTNET_TQ) {
- VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
- vq->vq_queue_index);
- VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
- vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
- } else {
- VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
- vq->vq_queue_index);
- VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
- vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
+#ifdef RTE_MACHINE_CPUFLAG_SSSE3
+ if (use_simple_rxtx) {
+ int mid_idx = vq->vq_nentries >> 1;
+ for (i = 0; i < mid_idx; i++) {
+ vq->vq_ring.avail->ring[i] = i + mid_idx;
+ vq->vq_ring.desc[i + mid_idx].next = i;
+ vq->vq_ring.desc[i + mid_idx].addr =
+ vq->virtio_net_hdr_mem +
+ mid_idx * vq->hw->vtnet_hdr_size;
+ vq->vq_ring.desc[i + mid_idx].len =
+ vq->hw->vtnet_hdr_size;
+ vq->vq_ring.desc[i + mid_idx].flags =
+ VRING_DESC_F_NEXT;
+ vq->vq_ring.desc[i].flags = 0;
+ }
+ for (i = mid_idx; i < vq->vq_nentries; i++)
+ vq->vq_ring.avail->ring[i] = i;
+ }
+#endif
}
}
ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
nb_desc, socket_id, &vq);
if (ret < 0) {
- PMD_INIT_LOG(ERR, "tvq initialization failed");
+ PMD_INIT_LOG(ERR, "rvq initialization failed");
return ret;
}
vq->mpool = mp;
dev->data->rx_queues[queue_idx] = vq;
+
+#ifdef RTE_MACHINE_CPUFLAG_SSSE3
+ virtio_rxq_vec_setup(vq);
+#endif
+
return 0;
}
+void
+virtio_dev_rx_queue_release(void *rxq)
+{
+ virtio_dev_queue_release(rxq);
+}
+
/*
* struct rte_eth_dev *dev: Used to update dev
* uint16_t nb_desc: Defaults to values read from config space
const struct rte_eth_txconf *tx_conf)
{
uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
+
+#ifdef RTE_MACHINE_CPUFLAG_SSSE3
+ struct virtio_hw *hw = dev->data->dev_private;
+#endif
struct virtqueue *vq;
uint16_t tx_free_thresh;
int ret;
return -EINVAL;
}
+#ifdef RTE_MACHINE_CPUFLAG_SSSE3
+ /* Use simple rx/tx func if single segment and no offloads */
+ if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS &&
+ !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ PMD_INIT_LOG(INFO, "Using simple rx/tx path");
+ dev->tx_pkt_burst = virtio_xmit_pkts_simple;
+ dev->rx_pkt_burst = virtio_recv_pkts_vec;
+ use_simple_rxtx = 1;
+ }
+#endif
+
ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
nb_desc, socket_id, &vq);
if (ret < 0) {
return 0;
}
+void
+virtio_dev_tx_queue_release(void *txq)
+{
+ virtio_dev_queue_release(txq);
+}
+
static void
virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
{
}
}
+static void
+virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf)
+{
+ uint32_t s = mbuf->pkt_len;
+ struct ether_addr *ea;
+
+ if (s == 64) {
+ vq->size_bins[1]++;
+ } else if (s > 64 && s < 1024) {
+ uint32_t bin;
+
+ /* count zeros, and offset into correct bin */
+ bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
+ vq->size_bins[bin]++;
+ } else {
+ if (s < 64)
+ vq->size_bins[0]++;
+ else if (s < 1519)
+ vq->size_bins[6]++;
+ else if (s >= 1519)
+ vq->size_bins[7]++;
+ }
+
+ ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
+ vq->multicast += is_multicast_ether_addr(ea);
+ vq->broadcast += is_broadcast_ether_addr(ea);
+}
+
#define VIRTIO_MBUF_BURST_SZ 64
#define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
uint16_t
struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
int error;
uint32_t i, nb_enqueued;
- const uint32_t hdr_size = sizeof(struct virtio_net_hdr);
+ uint32_t hdr_size;
nb_used = VIRTQUEUE_NUSED(rxvq);
hw = rxvq->hw;
nb_rx = 0;
nb_enqueued = 0;
+ hdr_size = hw->vtnet_hdr_size;
for (i = 0; i < num ; i++) {
rxm = rcv_pkts[i];
rxm->port = rxvq->port_id;
rxm->data_off = RTE_PKTMBUF_HEADROOM;
+ rxm->ol_flags = 0;
+ rxm->vlan_tci = 0;
rxm->nb_segs = 1;
rxm->next = NULL;
VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
rx_pkts[nb_rx++] = rxm;
+
rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
+ virtio_update_packet_stats(rxvq, rxm);
}
rxvq->packets += nb_rx;
uint32_t seg_num;
uint16_t extra_idx;
uint32_t seg_res;
- const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
+ uint32_t hdr_size;
nb_used = VIRTQUEUE_NUSED(rxvq);
seg_num = 0;
extra_idx = 0;
seg_res = 0;
+ hdr_size = hw->vtnet_hdr_size;
while (i < nb_used) {
struct virtio_net_hdr_mrg_rxbuf *header;
rxm->data_off = RTE_PKTMBUF_HEADROOM;
rxm->nb_segs = seg_num;
rxm->next = NULL;
+ rxm->ol_flags = 0;
+ rxm->vlan_tci = 0;
rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
rxm->data_len = (uint16_t)(len[0] - hdr_size);
rx_pkts[nb_rx]->data_len);
rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
+ virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]);
nb_rx++;
}
virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
struct virtqueue *txvq = tx_queue;
- struct rte_mbuf *txm;
uint16_t nb_used, nb_tx;
int error;
if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh))
virtio_xmit_cleanup(txvq, nb_used);
- nb_tx = 0;
-
- while (nb_tx < nb_pkts) {
+ for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+ struct rte_mbuf *txm = tx_pkts[nb_tx];
/* Need one more descriptor for virtio header. */
- int need = tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1;
+ int need = txm->nb_segs - txvq->vq_free_cnt + 1;
- /*Positive value indicates it need free vring descriptors */
+ /* Positive value indicates it need free vring descriptors */
if (unlikely(need > 0)) {
nb_used = VIRTQUEUE_NUSED(txvq);
virtio_rmb();
need = RTE_MIN(need, (int)nb_used);
virtio_xmit_cleanup(txvq, need);
- need = (int)tx_pkts[nb_tx]->nb_segs -
- txvq->vq_free_cnt + 1;
- }
-
- /*
- * Zero or negative value indicates it has enough free
- * descriptors to use for transmitting.
- */
- if (likely(need <= 0)) {
- txm = tx_pkts[nb_tx];
-
- /* Do VLAN tag insertion */
- if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
- error = rte_vlan_insert(&txm);
- if (unlikely(error)) {
- rte_pktmbuf_free(txm);
- ++nb_tx;
- continue;
- }
+ need = txm->nb_segs - txvq->vq_free_cnt + 1;
+ if (unlikely(need > 0)) {
+ PMD_TX_LOG(ERR,
+ "No free tx descriptors to transmit");
+ break;
}
+ }
- /* Enqueue Packet buffers */
- error = virtqueue_enqueue_xmit(txvq, txm);
+ /* Do VLAN tag insertion */
+ if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
+ error = rte_vlan_insert(&txm);
if (unlikely(error)) {
- if (error == ENOSPC)
- PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0");
- else if (error == EMSGSIZE)
- PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1");
- else
- PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error);
- break;
+ rte_pktmbuf_free(txm);
+ continue;
}
- nb_tx++;
- txvq->bytes += txm->pkt_len;
- } else {
- PMD_TX_LOG(ERR, "No free tx descriptors to transmit");
+ }
+
+ /* Enqueue Packet buffers */
+ error = virtqueue_enqueue_xmit(txvq, txm);
+ if (unlikely(error)) {
+ if (error == ENOSPC)
+ PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0");
+ else if (error == EMSGSIZE)
+ PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1");
+ else
+ PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error);
break;
}
+
+ txvq->bytes += txm->pkt_len;
+ virtio_update_packet_stats(txvq, txm);
}
txvq->packets += nb_tx;
git.droids-corp.org / dpdk.git / blobdiff