uint32_t desc_avail, desc_offset;
uint32_t mbuf_avail, mbuf_offset;
uint32_t cpy_len;
- uint64_t dlen;
+ uint64_t desc_chunck_len;
struct vring_desc *desc;
- uint64_t desc_addr;
+ uint64_t desc_addr, desc_gaddr;
/* A counter to avoid desc dead loop chain */
uint16_t nr_desc = 1;
struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
int error = 0;
desc = &descs[desc_idx];
- dlen = desc->len;
- desc_addr = vhost_iova_to_vva(dev, vq, desc->addr,
- &dlen, VHOST_ACCESS_RW);
+ desc_chunck_len = desc->len;
+ desc_gaddr = desc->addr;
+ desc_addr = vhost_iova_to_vva(dev, vq, desc_gaddr,
+ &desc_chunck_len, VHOST_ACCESS_RW);
/*
* Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
* performance issue with some versions of gcc (4.8.4 and 5.3.0) which
* otherwise stores offset on the stack instead of in a register.
*/
- if (unlikely(dlen != desc->len || desc->len < dev->vhost_hlen) ||
- !desc_addr) {
+ if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) {
error = -1;
goto out;
}
rte_prefetch0((void *)(uintptr_t)desc_addr);
- virtio_enqueue_offload(m, (struct virtio_net_hdr *)(uintptr_t)desc_addr);
- vhost_log_write(dev, desc->addr, dev->vhost_hlen);
- PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
+ if (likely(desc_chunck_len >= dev->vhost_hlen)) {
+ virtio_enqueue_offload(m,
+ (struct virtio_net_hdr *)(uintptr_t)desc_addr);
+ PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
+ vhost_log_write(dev, desc_gaddr, dev->vhost_hlen);
+ } else {
+ struct virtio_net_hdr vnet_hdr;
+ uint64_t remain = dev->vhost_hlen;
+ uint64_t len;
+ uint64_t src = (uint64_t)(uintptr_t)&vnet_hdr, dst;
+ uint64_t guest_addr = desc_gaddr;
+
+ virtio_enqueue_offload(m, &vnet_hdr);
+
+ while (remain) {
+ len = remain;
+ dst = vhost_iova_to_vva(dev, vq, guest_addr,
+ &len, VHOST_ACCESS_RW);
+ if (unlikely(!dst || !len)) {
+ error = -1;
+ goto out;
+ }
+
+ rte_memcpy((void *)(uintptr_t)dst,
+ (void *)(uintptr_t)src, len);
+
+ PRINT_PACKET(dev, (uintptr_t)dst, (uint32_t)len, 0);
+ vhost_log_write(dev, guest_addr, len);
+ remain -= len;
+ guest_addr += len;
+ dst += len;
+ }
+ }
- desc_offset = dev->vhost_hlen;
desc_avail = desc->len - dev->vhost_hlen;
+ if (unlikely(desc_chunck_len < dev->vhost_hlen)) {
+ desc_chunck_len = desc_avail;
+ desc_gaddr = desc->addr + dev->vhost_hlen;
+ desc_addr = vhost_iova_to_vva(dev,
+ vq, desc_gaddr,
+ &desc_chunck_len,
+ VHOST_ACCESS_RW);
+ if (unlikely(!desc_addr)) {
+ error = -1;
+ goto out;
+ }
+
+ desc_offset = 0;
+ } else {
+ desc_offset = dev->vhost_hlen;
+ desc_chunck_len -= dev->vhost_hlen;
+ }
mbuf_avail = rte_pktmbuf_data_len(m);
mbuf_offset = 0;
}
desc = &descs[desc->next];
- dlen = desc->len;
- desc_addr = vhost_iova_to_vva(dev, vq, desc->addr,
- &dlen,
+ desc_chunck_len = desc->len;
+ desc_gaddr = desc->addr;
+ desc_addr = vhost_iova_to_vva(dev, vq, desc_gaddr,
+ &desc_chunck_len,
VHOST_ACCESS_RW);
- if (unlikely(!desc_addr || dlen != desc->len)) {
+ if (unlikely(!desc_addr)) {
error = -1;
goto out;
}
desc_offset = 0;
desc_avail = desc->len;
+ } else if (unlikely(desc_chunck_len == 0)) {
+ desc_chunck_len = desc_avail;
+ desc_gaddr += desc_offset;
+ desc_addr = vhost_iova_to_vva(dev,
+ vq, desc_gaddr,
+ &desc_chunck_len, VHOST_ACCESS_RW);
+ if (unlikely(!desc_addr)) {
+ error = -1;
+ goto out;
+ }
+ desc_offset = 0;
}
- cpy_len = RTE_MIN(desc_avail, mbuf_avail);
+ cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) {
rte_memcpy((void *)((uintptr_t)(desc_addr +
desc_offset)),
rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
cpy_len);
- vhost_log_write(dev, desc->addr + desc_offset, cpy_len);
+ vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len);
PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
cpy_len, 0);
} else {
(void *)((uintptr_t)(desc_addr + desc_offset));
batch_copy[copy_nb].src =
rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
- batch_copy[copy_nb].log_addr = desc->addr + desc_offset;
+ batch_copy[copy_nb].log_addr = desc_gaddr + desc_offset;
batch_copy[copy_nb].len = cpy_len;
copy_nb++;
}
mbuf_offset += cpy_len;
desc_avail -= cpy_len;
desc_offset += cpy_len;
+ desc_chunck_len -= cpy_len;
}
out:
uint16_t num_buffers)
{
uint32_t vec_idx = 0;
- uint64_t desc_addr;
+ uint64_t desc_addr, desc_gaddr;
uint32_t mbuf_offset, mbuf_avail;
uint32_t desc_offset, desc_avail;
uint32_t cpy_len;
- uint64_t dlen;
+ uint64_t desc_chunck_len;
uint64_t hdr_addr, hdr_phys_addr;
struct rte_mbuf *hdr_mbuf;
struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
+ struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
uint16_t copy_nb = vq->batch_copy_nb_elems;
int error = 0;
goto out;
}
- dlen = buf_vec[vec_idx].buf_len;
- desc_addr = vhost_iova_to_vva(dev, vq, buf_vec[vec_idx].buf_addr,
- &dlen, VHOST_ACCESS_RW);
- if (dlen != buf_vec[vec_idx].buf_len ||
- buf_vec[vec_idx].buf_len < dev->vhost_hlen ||
- !desc_addr) {
+ desc_chunck_len = buf_vec[vec_idx].buf_len;
+ desc_gaddr = buf_vec[vec_idx].buf_addr;
+ desc_addr = vhost_iova_to_vva(dev, vq,
+ desc_gaddr,
+ &desc_chunck_len,
+ VHOST_ACCESS_RW);
+ if (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) {
error = -1;
goto out;
}
hdr_mbuf = m;
hdr_addr = desc_addr;
- hdr_phys_addr = buf_vec[vec_idx].buf_addr;
+ if (unlikely(desc_chunck_len < dev->vhost_hlen))
+ hdr = &tmp_hdr;
+ else
+ hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
+ hdr_phys_addr = desc_gaddr;
rte_prefetch0((void *)(uintptr_t)hdr_addr);
VHOST_LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
dev->vid, num_buffers);
desc_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
- desc_offset = dev->vhost_hlen;
+ if (unlikely(desc_chunck_len < dev->vhost_hlen)) {
+ desc_chunck_len = desc_avail;
+ desc_gaddr += dev->vhost_hlen;
+ desc_addr = vhost_iova_to_vva(dev, vq,
+ desc_gaddr,
+ &desc_chunck_len,
+ VHOST_ACCESS_RW);
+ if (unlikely(!desc_addr)) {
+ error = -1;
+ goto out;
+ }
+
+ desc_offset = 0;
+ } else {
+ desc_offset = dev->vhost_hlen;
+ desc_chunck_len -= dev->vhost_hlen;
+ }
+
mbuf_avail = rte_pktmbuf_data_len(m);
mbuf_offset = 0;
/* done with current desc buf, get the next one */
if (desc_avail == 0) {
vec_idx++;
- dlen = buf_vec[vec_idx].buf_len;
+ desc_chunck_len = buf_vec[vec_idx].buf_len;
+ desc_gaddr = buf_vec[vec_idx].buf_addr;
desc_addr =
vhost_iova_to_vva(dev, vq,
- buf_vec[vec_idx].buf_addr,
- &dlen,
+ desc_gaddr,
+ &desc_chunck_len,
VHOST_ACCESS_RW);
- if (unlikely(!desc_addr ||
- dlen != buf_vec[vec_idx].buf_len)) {
+ if (unlikely(!desc_addr)) {
error = -1;
goto out;
}
rte_prefetch0((void *)(uintptr_t)desc_addr);
desc_offset = 0;
desc_avail = buf_vec[vec_idx].buf_len;
+ } else if (unlikely(desc_chunck_len == 0)) {
+ desc_chunck_len = desc_avail;
+ desc_gaddr += desc_offset;
+ desc_addr = vhost_iova_to_vva(dev, vq,
+ desc_gaddr,
+ &desc_chunck_len, VHOST_ACCESS_RW);
+ if (unlikely(!desc_addr)) {
+ error = -1;
+ goto out;
+ }
+ desc_offset = 0;
}
/* done with current mbuf, get the next one */
}
if (hdr_addr) {
- struct virtio_net_hdr_mrg_rxbuf *hdr;
-
- hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)
- hdr_addr;
virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
ASSIGN_UNLESS_EQUAL(hdr->num_buffers, num_buffers);
- vhost_log_write(dev, hdr_phys_addr, dev->vhost_hlen);
- PRINT_PACKET(dev, (uintptr_t)hdr_addr,
- dev->vhost_hlen, 0);
+ if (unlikely(hdr == &tmp_hdr)) {
+ uint64_t len;
+ uint64_t remain = dev->vhost_hlen;
+ uint64_t src = (uint64_t)(uintptr_t)hdr, dst;
+ uint64_t guest_addr = hdr_phys_addr;
+
+ while (remain) {
+ len = remain;
+ dst = vhost_iova_to_vva(dev, vq,
+ guest_addr, &len,
+ VHOST_ACCESS_RW);
+ if (unlikely(!dst || !len)) {
+ error = -1;
+ goto out;
+ }
+
+ rte_memcpy((void *)(uintptr_t)dst,
+ (void *)(uintptr_t)src,
+ len);
+
+ PRINT_PACKET(dev, (uintptr_t)dst,
+ (uint32_t)len, 0);
+ vhost_log_write(dev, guest_addr, len);
+
+ remain -= len;
+ guest_addr += len;
+ dst += len;
+ }
+ } else {
+ PRINT_PACKET(dev, (uintptr_t)hdr_addr,
+ dev->vhost_hlen, 0);
+ vhost_log_write(dev, hdr_phys_addr,
+ dev->vhost_hlen);
+ }
hdr_addr = 0;
}
- cpy_len = RTE_MIN(desc_avail, mbuf_avail);
+ cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) {
rte_memcpy((void *)((uintptr_t)(desc_addr +
desc_offset)),
rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
cpy_len);
- vhost_log_write(dev,
- buf_vec[vec_idx].buf_addr + desc_offset,
- cpy_len);
+ vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len);
PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
cpy_len, 0);
} else {
(void *)((uintptr_t)(desc_addr + desc_offset));
batch_copy[copy_nb].src =
rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
- batch_copy[copy_nb].log_addr =
- buf_vec[vec_idx].buf_addr + desc_offset;
+ batch_copy[copy_nb].log_addr = desc_gaddr + desc_offset;
batch_copy[copy_nb].len = cpy_len;
copy_nb++;
}
mbuf_offset += cpy_len;
desc_avail -= cpy_len;
desc_offset += cpy_len;
+ desc_chunck_len -= cpy_len;
}
out:
struct rte_mempool *mbuf_pool)
{
struct vring_desc *desc;
- uint64_t desc_addr;
+ uint64_t desc_addr, desc_gaddr;
uint32_t desc_avail, desc_offset;
uint32_t mbuf_avail, mbuf_offset;
uint32_t cpy_len;
- uint64_t dlen;
+ uint64_t desc_chunck_len;
struct rte_mbuf *cur = m, *prev = m;
+ struct virtio_net_hdr tmp_hdr;
struct virtio_net_hdr *hdr = NULL;
/* A counter to avoid desc dead loop chain */
uint32_t nr_desc = 1;
goto out;
}
- dlen = desc->len;
+ desc_chunck_len = desc->len;
+ desc_gaddr = desc->addr;
desc_addr = vhost_iova_to_vva(dev,
- vq, desc->addr,
- &dlen,
+ vq, desc_gaddr,
+ &desc_chunck_len,
VHOST_ACCESS_RO);
- if (unlikely(!desc_addr || dlen != desc->len)) {
+ if (unlikely(!desc_addr)) {
error = -1;
goto out;
}
if (virtio_net_with_host_offload(dev)) {
- hdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr);
- rte_prefetch0(hdr);
+ if (unlikely(desc_chunck_len < sizeof(struct virtio_net_hdr))) {
+ uint64_t len = desc_chunck_len;
+ uint64_t remain = sizeof(struct virtio_net_hdr);
+ uint64_t src = desc_addr;
+ uint64_t dst = (uint64_t)(uintptr_t)&tmp_hdr;
+ uint64_t guest_addr = desc_gaddr;
+
+ /*
+ * No luck, the virtio-net header doesn't fit
+ * in a contiguous virtual area.
+ */
+ while (remain) {
+ len = remain;
+ src = vhost_iova_to_vva(dev, vq,
+ guest_addr, &len,
+ VHOST_ACCESS_RO);
+ if (unlikely(!src || !len)) {
+ error = -1;
+ goto out;
+ }
+
+ rte_memcpy((void *)(uintptr_t)dst,
+ (void *)(uintptr_t)src, len);
+
+ guest_addr += len;
+ remain -= len;
+ dst += len;
+ }
+
+ hdr = &tmp_hdr;
+ } else {
+ hdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr);
+ rte_prefetch0(hdr);
+ }
}
/*
goto out;
}
- dlen = desc->len;
+ desc_chunck_len = desc->len;
+ desc_gaddr = desc->addr;
desc_addr = vhost_iova_to_vva(dev,
- vq, desc->addr,
- &dlen,
+ vq, desc_gaddr,
+ &desc_chunck_len,
VHOST_ACCESS_RO);
- if (unlikely(!desc_addr || dlen != desc->len)) {
+ if (unlikely(!desc_addr)) {
error = -1;
goto out;
}
nr_desc += 1;
} else {
desc_avail = desc->len - dev->vhost_hlen;
- desc_offset = dev->vhost_hlen;
+
+ if (unlikely(desc_chunck_len < dev->vhost_hlen)) {
+ desc_chunck_len = desc_avail;
+ desc_gaddr += dev->vhost_hlen;
+ desc_addr = vhost_iova_to_vva(dev,
+ vq, desc_gaddr,
+ &desc_chunck_len,
+ VHOST_ACCESS_RO);
+ if (unlikely(!desc_addr)) {
+ error = -1;
+ goto out;
+ }
+
+ desc_offset = 0;
+ } else {
+ desc_offset = dev->vhost_hlen;
+ desc_chunck_len -= dev->vhost_hlen;
+ }
}
rte_prefetch0((void *)(uintptr_t)(desc_addr + desc_offset));
- PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), desc_avail, 0);
+ PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
+ (uint32_t)desc_chunck_len, 0);
mbuf_offset = 0;
mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM;
while (1) {
uint64_t hpa;
- cpy_len = RTE_MIN(desc_avail, mbuf_avail);
+ cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
/*
* A desc buf might across two host physical pages that are
* will be copied even though zero copy is enabled.
*/
if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,
- desc->addr + desc_offset, cpy_len)))) {
+ desc_gaddr + desc_offset, cpy_len)))) {
cur->data_len = cpy_len;
cur->data_off = 0;
cur->buf_addr = (void *)(uintptr_t)(desc_addr
} else {
if (likely(cpy_len > MAX_BATCH_LEN ||
copy_nb >= vq->size ||
- (hdr && cur == m))) {
+ (hdr && cur == m) ||
+ desc->len != desc_chunck_len)) {
rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
mbuf_offset),
(void *)((uintptr_t)(desc_addr +
mbuf_avail -= cpy_len;
mbuf_offset += cpy_len;
desc_avail -= cpy_len;
+ desc_chunck_len -= cpy_len;
desc_offset += cpy_len;
/* This desc reaches to its end, get the next one */
goto out;
}
- dlen = desc->len;
+ desc_chunck_len = desc->len;
+ desc_gaddr = desc->addr;
desc_addr = vhost_iova_to_vva(dev,
- vq, desc->addr,
- &dlen, VHOST_ACCESS_RO);
- if (unlikely(!desc_addr || dlen != desc->len)) {
+ vq, desc_gaddr,
+ &desc_chunck_len,
+ VHOST_ACCESS_RO);
+ if (unlikely(!desc_addr)) {
error = -1;
goto out;
}
desc_offset = 0;
desc_avail = desc->len;
- PRINT_PACKET(dev, (uintptr_t)desc_addr, desc->len, 0);
+ PRINT_PACKET(dev, (uintptr_t)desc_addr,
+ (uint32_t)desc_chunck_len, 0);
+ } else if (unlikely(desc_chunck_len == 0)) {
+ desc_chunck_len = desc_avail;
+ desc_gaddr += desc_offset;
+ desc_addr = vhost_iova_to_vva(dev, vq,
+ desc_gaddr,
+ &desc_chunck_len,
+ VHOST_ACCESS_RO);
+ if (unlikely(!desc_addr)) {
+ error = -1;
+ goto out;
+ }
+ desc_offset = 0;
+
+ PRINT_PACKET(dev, (uintptr_t)desc_addr,
+ (uint32_t)desc_chunck_len, 0);
}
/*