#include <rte_mbuf.h>
#include <rte_memcpy.h>
+#include <rte_net.h>
#include <rte_ether.h>
#include <rte_ip.h>
#include <rte_vhost.h>
{
uint16_t i;
uint16_t flags;
- uint16_t last_used_idx = vq->last_used_idx;
- struct vring_packed_desc *desc_base = &vq->desc_packed[last_used_idx];
+ uint16_t last_used_idx;
+ struct vring_packed_desc *desc_base;
if (vq->shadow_used_idx) {
do_data_copy_enqueue(dev, vq);
vhost_flush_enqueue_shadow_packed(dev, vq);
}
+ last_used_idx = vq->last_used_idx;
+ desc_base = &vq->desc_packed[last_used_idx];
+
flags = PACKED_DESC_ENQUEUE_USED_FLAG(vq->used_wrap_counter);
vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
store_dma_desc_info_split(struct vring_used_elem *s_ring, struct vring_used_elem *d_ring,
uint16_t ring_size, uint16_t s_idx, uint16_t d_idx, uint16_t count)
{
- uint16_t elem_size = sizeof(struct vring_used_elem);
+ size_t elem_size = sizeof(struct vring_used_elem);
if (d_idx + count <= ring_size) {
rte_memcpy(d_ring + d_idx, s_ring + s_idx, count * elem_size);
struct vring_used_elem_packed *d_ring,
uint16_t ring_size, uint16_t s_idx, uint16_t d_idx, uint16_t count)
{
- uint16_t elem_size = sizeof(struct vring_used_elem_packed);
+ size_t elem_size = sizeof(struct vring_used_elem_packed);
if (d_idx + count <= ring_size) {
rte_memcpy(d_ring + d_idx, s_ring + s_idx, count * elem_size);
vq->desc_packed[head_idx].flags = head_flags;
}
+static __rte_always_inline int
+virtio_dev_rx_async_batch_packed(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts,
+ struct rte_mbuf **comp_pkts, uint32_t *pkt_done)
+{
+ uint16_t i;
+ uint32_t cpy_threshold = vq->async_threshold;
+
+ vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+ if (unlikely(pkts[i]->pkt_len >= cpy_threshold))
+ return -1;
+ }
+ if (!virtio_dev_rx_batch_packed(dev, vq, pkts)) {
+ vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
+ comp_pkts[(*pkt_done)++] = pkts[i];
+
+ return 0;
+ }
+
+ return -1;
+}
+
static __rte_always_inline int
vhost_enqueue_async_single_packed(struct virtio_net *dev,
struct vhost_virtqueue *vq,
struct rte_mbuf **comp_pkts, uint32_t *comp_count)
{
uint32_t pkt_idx = 0, pkt_burst_idx = 0;
+ uint32_t remained = count;
uint16_t async_descs_idx = 0;
uint16_t num_buffers;
uint16_t num_descs;
uint32_t num_async_pkts = 0, num_done_pkts = 0;
struct vring_packed_desc async_descs[vq->size];
- rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
+ do {
+ rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
+ if (remained >= PACKED_BATCH_SIZE) {
+ if (!virtio_dev_rx_async_batch_packed(dev, vq,
+ &pkts[pkt_idx], comp_pkts, &num_done_pkts)) {
+ pkt_idx += PACKED_BATCH_SIZE;
+ remained -= PACKED_BATCH_SIZE;
+ continue;
+ }
+ }
- for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
num_buffers = 0;
num_descs = 0;
-
if (unlikely(virtio_dev_rx_async_single_packed(dev, vq, pkts[pkt_idx],
&num_descs, &num_buffers,
&async_descs[async_descs_idx],
comp_pkts[num_done_pkts++] = pkts[pkt_idx];
}
+ pkt_idx++;
+ remained--;
vq_inc_last_avail_packed(vq, num_descs);
/*
*/
pkt_err = pkt_burst_idx - n_pkts;
pkt_burst_idx = 0;
- pkt_idx++;
break;
}
pkt_burst_idx = 0;
}
- }
+ } while (pkt_idx < count);
if (pkt_burst_idx) {
n_pkts = vq->async_ops.transfer_data(dev->vid, queue_id, tdes, 0, pkt_burst_idx);
}
static __rte_always_inline void
-vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
+vhost_dequeue_offload_legacy(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
{
uint16_t l4_proto = 0;
void *l4_hdr = NULL;
struct rte_tcp_hdr *tcp_hdr = NULL;
- if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
- return;
-
parse_ethernet(m, &l4_proto, &l4_hdr);
if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
if (hdr->csum_start == (m->l2_len + m->l3_len)) {
}
}
+static __rte_always_inline void
+vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m,
+ bool legacy_ol_flags)
+{
+ struct rte_net_hdr_lens hdr_lens;
+ int l4_supported = 0;
+ uint32_t ptype;
+
+ if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
+ return;
+
+ if (legacy_ol_flags) {
+ vhost_dequeue_offload_legacy(hdr, m);
+ return;
+ }
+
+ m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
+
+ ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
+ m->packet_type = ptype;
+ if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP ||
+ (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP ||
+ (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP)
+ l4_supported = 1;
+
+ /* According to Virtio 1.1 spec, the device only needs to look at
+ * VIRTIO_NET_HDR_F_NEEDS_CSUM in the packet transmission path.
+ * This differs from the processing incoming packets path where the
+ * driver could rely on VIRTIO_NET_HDR_F_DATA_VALID flag set by the
+ * device.
+ *
+ * 5.1.6.2.1 Driver Requirements: Packet Transmission
+ * The driver MUST NOT set the VIRTIO_NET_HDR_F_DATA_VALID and
+ * VIRTIO_NET_HDR_F_RSC_INFO bits in flags.
+ *
+ * 5.1.6.2.2 Device Requirements: Packet Transmission
+ * The device MUST ignore flag bits that it does not recognize.
+ */
+ if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+ uint32_t hdrlen;
+
+ hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len;
+ if (hdr->csum_start <= hdrlen && l4_supported != 0) {
+ m->ol_flags |= PKT_RX_L4_CKSUM_NONE;
+ } else {
+ /* Unknown proto or tunnel, do sw cksum. We can assume
+ * the cksum field is in the first segment since the
+ * buffers we provided to the host are large enough.
+ * In case of SCTP, this will be wrong since it's a CRC
+ * but there's nothing we can do.
+ */
+ uint16_t csum = 0, off;
+
+ if (rte_raw_cksum_mbuf(m, hdr->csum_start,
+ rte_pktmbuf_pkt_len(m) - hdr->csum_start, &csum) < 0)
+ return;
+ if (likely(csum != 0xffff))
+ csum = ~csum;
+ off = hdr->csum_offset + hdr->csum_start;
+ if (rte_pktmbuf_data_len(m) >= off + 1)
+ *rte_pktmbuf_mtod_offset(m, uint16_t *, off) = csum;
+ }
+ }
+
+ if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
+ if (hdr->gso_size == 0)
+ return;
+
+ switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
+ case VIRTIO_NET_HDR_GSO_TCPV4:
+ case VIRTIO_NET_HDR_GSO_TCPV6:
+ if ((ptype & RTE_PTYPE_L4_MASK) != RTE_PTYPE_L4_TCP)
+ break;
+ m->ol_flags |= PKT_RX_LRO | PKT_RX_L4_CKSUM_NONE;
+ m->tso_segsz = hdr->gso_size;
+ break;
+ case VIRTIO_NET_HDR_GSO_UDP:
+ if ((ptype & RTE_PTYPE_L4_MASK) != RTE_PTYPE_L4_UDP)
+ break;
+ m->ol_flags |= PKT_RX_LRO | PKT_RX_L4_CKSUM_NONE;
+ m->tso_segsz = hdr->gso_size;
+ break;
+ default:
+ break;
+ }
+ }
+}
+
static __rte_noinline void
copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr,
struct buf_vector *buf_vec)
static __rte_always_inline int
copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
struct buf_vector *buf_vec, uint16_t nr_vec,
- struct rte_mbuf *m, struct rte_mempool *mbuf_pool)
+ struct rte_mbuf *m, struct rte_mempool *mbuf_pool,
+ bool legacy_ol_flags)
{
uint32_t buf_avail, buf_offset;
uint64_t buf_addr, buf_len;
m->pkt_len += mbuf_offset;
if (hdr)
- vhost_dequeue_offload(hdr, m);
+ vhost_dequeue_offload(hdr, m, legacy_ol_flags);
out:
return 0;
}
+/*
+ * Prepare a host supported pktmbuf.
+ */
static __rte_always_inline int
virtio_dev_pktmbuf_prep(struct virtio_net *dev, struct rte_mbuf *pkt,
uint32_t data_len)
return -1;
}
-/*
- * Allocate a host supported pktmbuf.
- */
-static __rte_always_inline struct rte_mbuf *
-virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct rte_mempool *mp,
- uint32_t data_len)
-{
- struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp);
-
- if (unlikely(pkt == NULL)) {
- VHOST_LOG_DATA(ERR,
- "Failed to allocate memory for mbuf.\n");
- return NULL;
- }
-
- if (virtio_dev_pktmbuf_prep(dev, pkt, data_len)) {
- /* Data doesn't fit into the buffer and the host supports
- * only linear buffers
- */
- rte_pktmbuf_free(pkt);
- return NULL;
- }
-
- return pkt;
-}
-
-static __rte_noinline uint16_t
+__rte_always_inline
+static uint16_t
virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
- struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+ struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+ bool legacy_ol_flags)
{
uint16_t i;
uint16_t free_entries;
VHOST_LOG_DATA(DEBUG, "(%d) about to dequeue %u buffers\n",
dev->vid, count);
+ if (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts, count))
+ return 0;
+
for (i = 0; i < count; i++) {
struct buf_vector buf_vec[BUF_VECTOR_MAX];
uint16_t head_idx;
update_shadow_used_ring_split(vq, head_idx, 0);
- pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
- if (unlikely(pkts[i] == NULL)) {
+ err = virtio_dev_pktmbuf_prep(dev, pkts[i], buf_len);
+ if (unlikely(err)) {
/*
* mbuf allocation fails for jumbo packets when external
* buffer allocation is not allowed and linear buffer
}
err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
- mbuf_pool);
+ mbuf_pool, legacy_ol_flags);
if (unlikely(err)) {
- rte_pktmbuf_free(pkts[i]);
if (!allocerr_warned) {
VHOST_LOG_DATA(ERR,
"Failed to copy desc to mbuf on %s.\n",
}
}
+ if (dropped)
+ rte_pktmbuf_free_bulk(&pkts[i - 1], count - i + 1);
+
vq->last_avail_idx += i;
do_data_copy_dequeue(vq);
return (i - dropped);
}
+__rte_noinline
+static uint16_t
+virtio_dev_tx_split_legacy(struct virtio_net *dev,
+ struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **pkts, uint16_t count)
+{
+ return virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count, true);
+}
+
+__rte_noinline
+static uint16_t
+virtio_dev_tx_split_compliant(struct virtio_net *dev,
+ struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **pkts, uint16_t count)
+{
+ return virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count, false);
+}
+
static __rte_always_inline int
vhost_reserve_avail_batch_packed(struct virtio_net *dev,
struct vhost_virtqueue *vq,
static __rte_always_inline int
virtio_dev_tx_batch_packed(struct virtio_net *dev,
struct vhost_virtqueue *vq,
- struct rte_mbuf **pkts)
+ struct rte_mbuf **pkts,
+ bool legacy_ol_flags)
{
uint16_t avail_idx = vq->last_avail_idx;
uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);
if (virtio_net_with_host_offload(dev)) {
vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
hdr = (struct virtio_net_hdr *)(desc_addrs[i]);
- vhost_dequeue_offload(hdr, pkts[i]);
+ vhost_dequeue_offload(hdr, pkts[i], legacy_ol_flags);
}
}
struct rte_mempool *mbuf_pool,
struct rte_mbuf *pkts,
uint16_t *buf_id,
- uint16_t *desc_count)
+ uint16_t *desc_count,
+ bool legacy_ol_flags)
{
struct buf_vector buf_vec[BUF_VECTOR_MAX];
uint32_t buf_len;
}
err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts,
- mbuf_pool);
+ mbuf_pool, legacy_ol_flags);
if (unlikely(err)) {
if (!allocerr_warned) {
VHOST_LOG_DATA(ERR,
virtio_dev_tx_single_packed(struct virtio_net *dev,
struct vhost_virtqueue *vq,
struct rte_mempool *mbuf_pool,
- struct rte_mbuf *pkts)
+ struct rte_mbuf *pkts,
+ bool legacy_ol_flags)
{
uint16_t buf_id, desc_count = 0;
int ret;
ret = vhost_dequeue_single_packed(dev, vq, mbuf_pool, pkts, &buf_id,
- &desc_count);
+ &desc_count, legacy_ol_flags);
if (likely(desc_count > 0)) {
if (virtio_net_is_inorder(dev))
return ret;
}
-static __rte_noinline uint16_t
+__rte_always_inline
+static uint16_t
virtio_dev_tx_packed(struct virtio_net *dev,
struct vhost_virtqueue *__rte_restrict vq,
struct rte_mempool *mbuf_pool,
struct rte_mbuf **__rte_restrict pkts,
- uint32_t count)
+ uint32_t count,
+ bool legacy_ol_flags)
{
uint32_t pkt_idx = 0;
if (count - pkt_idx >= PACKED_BATCH_SIZE) {
if (!virtio_dev_tx_batch_packed(dev, vq,
- &pkts[pkt_idx])) {
+ &pkts[pkt_idx],
+ legacy_ol_flags)) {
pkt_idx += PACKED_BATCH_SIZE;
continue;
}
}
if (virtio_dev_tx_single_packed(dev, vq, mbuf_pool,
- pkts[pkt_idx]))
+ pkts[pkt_idx],
+ legacy_ol_flags))
break;
pkt_idx++;
} while (pkt_idx < count);
return pkt_idx;
}
+__rte_noinline
+static uint16_t
+virtio_dev_tx_packed_legacy(struct virtio_net *dev,
+ struct vhost_virtqueue *__rte_restrict vq, struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **__rte_restrict pkts, uint32_t count)
+{
+ return virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count, true);
+}
+
+__rte_noinline
+static uint16_t
+virtio_dev_tx_packed_compliant(struct virtio_net *dev,
+ struct vhost_virtqueue *__rte_restrict vq, struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **__rte_restrict pkts, uint32_t count)
+{
+ return virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count, false);
+}
+
uint16_t
rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
count -= 1;
}
- if (vq_is_packed(dev))
- count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count);
- else
- count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);
+ if (vq_is_packed(dev)) {
+ if (dev->flags & VIRTIO_DEV_LEGACY_OL_FLAGS)
+ count = virtio_dev_tx_packed_legacy(dev, vq, mbuf_pool, pkts, count);
+ else
+ count = virtio_dev_tx_packed_compliant(dev, vq, mbuf_pool, pkts, count);
+ } else {
+ if (dev->flags & VIRTIO_DEV_LEGACY_OL_FLAGS)
+ count = virtio_dev_tx_split_legacy(dev, vq, mbuf_pool, pkts, count);
+ else
+ count = virtio_dev_tx_split_compliant(dev, vq, mbuf_pool, pkts, count);
+ }
out:
if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
git.droids-corp.org / dpdk.git / blobdiff