X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvirtio_net.c;h=8ad30c94a6d1519660f3bae39d056d21c4fc0462;hb=3370975b994d793a89812031d86534916067dd67;hp=904981fd358fee3157cd44248b34c5b36783ad55;hpb=a922401f35cc40dead57cbd6e330a14b438905bf;p=dpdk.git diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 904981fd35..8ad30c94a6 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -122,7 +122,7 @@ flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq) static __rte_always_inline void update_shadow_used_ring_split(struct vhost_virtqueue *vq, - uint16_t desc_idx, uint16_t len) + uint16_t desc_idx, uint32_t len) { uint16_t i = vq->shadow_used_idx++; @@ -186,7 +186,7 @@ flush_shadow_used_ring_packed(struct virtio_net *dev, static __rte_always_inline void update_shadow_used_ring_packed(struct vhost_virtqueue *vq, - uint16_t desc_idx, uint16_t len, uint16_t count) + uint16_t desc_idx, uint32_t len, uint16_t count) { uint16_t i = vq->shadow_used_idx++; @@ -329,7 +329,7 @@ static __rte_always_inline int fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq, uint32_t avail_idx, uint16_t *vec_idx, struct buf_vector *buf_vec, uint16_t *desc_chain_head, - uint16_t *desc_chain_len, uint8_t perm) + uint32_t *desc_chain_len, uint8_t perm) { uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)]; uint16_t vec_id = *vec_idx; @@ -409,19 +409,26 @@ reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq, uint16_t max_tries, tries = 0; uint16_t head_idx = 0; - uint16_t len = 0; + uint32_t len = 0; *num_buffers = 0; cur_idx = vq->last_avail_idx; if (rxvq_is_mergeable(dev)) - max_tries = vq->size; + max_tries = vq->size - 1; else max_tries = 1; while (size > 0) { if (unlikely(cur_idx == avail_head)) return -1; + /* + * if we tried all available ring items, and still + * can't get enough buf, it means something abnormal + * happened. + */ + if (unlikely(++tries > max_tries)) + return -1; if (unlikely(fill_vec_buf_split(dev, vq, cur_idx, &vec_idx, buf_vec, @@ -433,16 +440,7 @@ reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq, size -= len; cur_idx++; - tries++; *num_buffers += 1; - - /* - * if we tried all available ring items, and still - * can't get enough buf, it means something abnormal - * happened. - */ - if (unlikely(tries > max_tries)) - return -1; } *nr_vec = vec_idx; @@ -454,7 +452,7 @@ static __rte_always_inline int fill_vec_buf_packed_indirect(struct virtio_net *dev, struct vhost_virtqueue *vq, struct vring_packed_desc *desc, uint16_t *vec_idx, - struct buf_vector *buf_vec, uint16_t *len, uint8_t perm) + struct buf_vector *buf_vec, uint32_t *len, uint8_t perm) { uint16_t i; uint32_t nr_descs; @@ -510,7 +508,7 @@ static __rte_always_inline int fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, uint16_t avail_idx, uint16_t *desc_count, struct buf_vector *buf_vec, uint16_t *vec_idx, - uint16_t *buf_id, uint16_t *len, uint8_t perm) + uint16_t *buf_id, uint32_t *len, uint8_t perm) { bool wrap_counter = vq->avail_wrap_counter; struct vring_packed_desc *descs = vq->desc_packed; @@ -523,6 +521,7 @@ fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, return -1; *desc_count = 0; + *len = 0; while (1) { if (unlikely(vec_id >= BUF_VECTOR_MAX)) @@ -575,18 +574,26 @@ reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, uint16_t max_tries, tries = 0; uint16_t buf_id = 0; - uint16_t len = 0; + uint32_t len = 0; uint16_t desc_count; *num_buffers = 0; avail_idx = vq->last_avail_idx; if (rxvq_is_mergeable(dev)) - max_tries = vq->size; + max_tries = vq->size - 1; else max_tries = 1; while (size > 0) { + /* + * if we tried all available ring items, and still + * can't get enough buf, it means something abnormal + * happened. + */ + if (unlikely(++tries > max_tries)) + return -1; + if (unlikely(fill_vec_buf_packed(dev, vq, avail_idx, &desc_count, buf_vec, &vec_idx, @@ -603,16 +610,7 @@ reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, avail_idx -= vq->size; *nr_descs += desc_count; - tries++; *num_buffers += 1; - - /* - * if we tried all available ring items, and still - * can't get enough buf, it means something abnormal - * happened. - */ - if (unlikely(tries > max_tries)) - return -1; } *nr_vec = vec_idx; @@ -720,7 +718,8 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, uint16_t hdr_vec_idx = 0; while (remain) { - len = remain; + len = RTE_MIN(remain, + buf_vec[hdr_vec_idx].buf_len); dst = buf_vec[hdr_vec_idx].buf_addr; rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, @@ -747,7 +746,7 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, hdr_addr = 0; } - cpy_len = RTE_MIN(buf_len, mbuf_avail); + cpy_len = RTE_MIN(buf_avail, mbuf_avail); if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { @@ -826,7 +825,7 @@ virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, if (likely(vq->shadow_used_idx)) { flush_shadow_used_ring_split(dev, vq); - vhost_vring_call(dev, vq); + vhost_vring_call_split(dev, vq); } return pkt_idx; @@ -879,7 +878,7 @@ virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, if (likely(vq->shadow_used_idx)) { flush_shadow_used_ring_packed(dev, vq); - vhost_vring_call(dev, vq); + vhost_vring_call_packed(dev, vq); } return pkt_idx; @@ -890,6 +889,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, uint32_t count) { struct vhost_virtqueue *vq; + uint32_t nb_tx = 0; VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) { @@ -917,9 +917,9 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, goto out; if (vq_is_packed(dev)) - count = virtio_dev_rx_packed(dev, vq, pkts, count); + nb_tx = virtio_dev_rx_packed(dev, vq, pkts, count); else - count = virtio_dev_rx_split(dev, vq, pkts, count); + nb_tx = virtio_dev_rx_split(dev, vq, pkts, count); out: if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) @@ -928,7 +928,7 @@ out: out_access_unlock: rte_spinlock_unlock(&vq->access_lock); - return count; + return nb_tx; } uint16_t @@ -1112,7 +1112,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, * in a contiguous virtual area. */ while (remain) { - len = remain; + len = RTE_MIN(remain, + buf_vec[hdr_vec_idx].buf_len); src = buf_vec[hdr_vec_idx].buf_addr; rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); @@ -1343,7 +1344,6 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, if (unlikely(dev->dequeue_zero_copy)) { struct zcopy_mbuf *zmbuf, *next; - int nr_updated = 0; for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); zmbuf != NULL; zmbuf = next) { @@ -1352,8 +1352,6 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, if (mbuf_is_consumed(zmbuf->mbuf)) { update_shadow_used_ring_split(vq, zmbuf->desc_idx, 0); - nr_updated += 1; - TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); restore_mbuf(zmbuf->mbuf); rte_pktmbuf_free(zmbuf->mbuf); @@ -1362,8 +1360,10 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, } } - flush_shadow_used_ring_split(dev, vq); - vhost_vring_call(dev, vq); + if (likely(vq->shadow_used_idx)) { + flush_shadow_used_ring_split(dev, vq); + vhost_vring_call_split(dev, vq); + } } rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); @@ -1382,7 +1382,8 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, for (i = 0; i < count; i++) { struct buf_vector buf_vec[BUF_VECTOR_MAX]; - uint16_t head_idx, dummy_len; + uint16_t head_idx; + uint32_t dummy_len; uint16_t nr_vec = 0; int err; @@ -1441,8 +1442,129 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, do_data_copy_dequeue(vq); if (unlikely(i < count)) vq->shadow_used_idx = i; - flush_shadow_used_ring_split(dev, vq); - vhost_vring_call(dev, vq); + if (likely(vq->shadow_used_idx)) { + flush_shadow_used_ring_split(dev, vq); + vhost_vring_call_split(dev, vq); + } + } + + return i; +} + +static __rte_always_inline uint16_t +virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) +{ + uint16_t i; + + rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]); + + if (unlikely(dev->dequeue_zero_copy)) { + struct zcopy_mbuf *zmbuf, *next; + + for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); + zmbuf != NULL; zmbuf = next) { + next = TAILQ_NEXT(zmbuf, next); + + if (mbuf_is_consumed(zmbuf->mbuf)) { + update_shadow_used_ring_packed(vq, + zmbuf->desc_idx, + 0, + zmbuf->desc_count); + + TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); + restore_mbuf(zmbuf->mbuf); + rte_pktmbuf_free(zmbuf->mbuf); + put_zmbuf(zmbuf); + vq->nr_zmbuf -= 1; + } + } + + if (likely(vq->shadow_used_idx)) { + flush_shadow_used_ring_packed(dev, vq); + vhost_vring_call_packed(dev, vq); + } + } + + VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); + + count = RTE_MIN(count, MAX_PKT_BURST); + VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n", + dev->vid, count); + + for (i = 0; i < count; i++) { + struct buf_vector buf_vec[BUF_VECTOR_MAX]; + uint16_t buf_id; + uint32_t dummy_len; + uint16_t desc_count, nr_vec = 0; + int err; + + if (unlikely(fill_vec_buf_packed(dev, vq, + vq->last_avail_idx, &desc_count, + buf_vec, &nr_vec, + &buf_id, &dummy_len, + VHOST_ACCESS_RW) < 0)) + break; + + if (likely(dev->dequeue_zero_copy == 0)) + update_shadow_used_ring_packed(vq, buf_id, 0, + desc_count); + + rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr); + + pkts[i] = rte_pktmbuf_alloc(mbuf_pool); + if (unlikely(pkts[i] == NULL)) { + RTE_LOG(ERR, VHOST_DATA, + "Failed to allocate memory for mbuf.\n"); + break; + } + + err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i], + mbuf_pool); + if (unlikely(err)) { + rte_pktmbuf_free(pkts[i]); + break; + } + + if (unlikely(dev->dequeue_zero_copy)) { + struct zcopy_mbuf *zmbuf; + + zmbuf = get_zmbuf(vq); + if (!zmbuf) { + rte_pktmbuf_free(pkts[i]); + break; + } + zmbuf->mbuf = pkts[i]; + zmbuf->desc_idx = buf_id; + zmbuf->desc_count = desc_count; + + /* + * Pin lock the mbuf; we will check later to see + * whether the mbuf is freed (when we are the last + * user) or not. If that's the case, we then could + * update the used ring safely. + */ + rte_mbuf_refcnt_update(pkts[i], 1); + + vq->nr_zmbuf += 1; + TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next); + } + + vq->last_avail_idx += desc_count; + if (vq->last_avail_idx >= vq->size) { + vq->last_avail_idx -= vq->size; + vq->avail_wrap_counter ^= 1; + } + } + + if (likely(dev->dequeue_zero_copy == 0)) { + do_data_copy_dequeue(vq); + if (unlikely(i < count)) + vq->shadow_used_idx = i; + if (likely(vq->shadow_used_idx)) { + flush_shadow_used_ring_packed(dev, vq); + vhost_vring_call_packed(dev, vq); + } } return i; @@ -1478,15 +1600,19 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id, if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0)) return 0; - if (unlikely(vq->enabled == 0)) + if (unlikely(vq->enabled == 0)) { + count = 0; goto out_access_unlock; + } if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) vhost_user_iotlb_rd_lock(vq); if (unlikely(vq->access_ok == 0)) - if (unlikely(vring_translate(dev, vq) < 0)) + if (unlikely(vring_translate(dev, vq) < 0)) { + count = 0; goto out; + } /* * Construct a RARP broadcast packet, and inject it to the "pkts" @@ -1511,12 +1637,16 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id, if (rarp_mbuf == NULL) { RTE_LOG(ERR, VHOST_DATA, "Failed to make RARP packet.\n"); - return 0; + count = 0; + goto out; } count -= 1; } - count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count); + if (vq_is_packed(dev)) + count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count); + else + count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count); out: if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))