X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost_rxtx.c;h=08a73fd04b541a598c44c4eda3f567790c6ba89d;hb=c830cb295411fc71c3f3d3591308d531937dd58f;hp=52f87c1da5d740131407b7825a12d8c6efa8cc27;hpb=7f74b95c444f617ac36c77253ade1bbb8dea4c88;p=dpdk.git diff --git a/lib/librte_vhost/vhost_rxtx.c b/lib/librte_vhost/vhost_rxtx.c index 52f87c1da5..08a73fd04b 100644 --- a/lib/librte_vhost/vhost_rxtx.c +++ b/lib/librte_vhost/vhost_rxtx.c @@ -147,10 +147,15 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0}; desc = &vq->desc[desc_idx]; - if (unlikely(desc->len < dev->vhost_hlen)) + desc_addr = gpa_to_vva(dev, desc->addr); + /* + * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid + * performance issue with some versions of gcc (4.8.4 and 5.3.0) which + * otherwise stores offset on the stack instead of in a register. + */ + if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) return -1; - desc_addr = gpa_to_vva(dev, desc->addr); rte_prefetch0((void *)(uintptr_t)desc_addr); virtio_enqueue_offload(m, &virtio_hdr.hdr); @@ -182,7 +187,10 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, return -1; desc = &vq->desc[desc->next]; - desc_addr = gpa_to_vva(dev, desc->addr); + desc_addr = gpa_to_vva(dev, desc->addr); + if (unlikely(!desc_addr)) + return -1; + desc_offset = 0; desc_avail = desc->len; } @@ -204,49 +212,6 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; } -/* - * As many data cores may want to access available buffers - * they need to be reserved. - */ -static inline uint32_t -reserve_avail_buf(struct vhost_virtqueue *vq, uint32_t count, - uint16_t *start, uint16_t *end) -{ - uint16_t res_start_idx; - uint16_t res_end_idx; - uint16_t avail_idx; - uint16_t free_entries; - int success; - - count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST); - -again: - res_start_idx = vq->last_used_idx_res; - avail_idx = *((volatile uint16_t *)&vq->avail->idx); - - free_entries = avail_idx - res_start_idx; - count = RTE_MIN(count, free_entries); - if (count == 0) - return 0; - - res_end_idx = res_start_idx + count; - - /* - * update vq->last_used_idx_res atomically; try again if failed. - * - * TODO: Allow to disable cmpset if no concurrency in application. - */ - success = rte_atomic16_cmpset(&vq->last_used_idx_res, - res_start_idx, res_end_idx); - if (unlikely(!success)) - goto again; - - *start = res_start_idx; - *end = res_end_idx; - - return count; -} - /** * This function adds buffers to the virtio devices RX virtqueue. Buffers can * be received from the physical port or from another virtio device. A packet @@ -259,7 +224,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, uint32_t count) { struct vhost_virtqueue *vq; - uint16_t res_start_idx, res_end_idx; + uint16_t avail_idx, free_entries, start_idx; uint16_t desc_indexes[MAX_PKT_BURST]; uint16_t used_idx; uint32_t i; @@ -275,17 +240,21 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, if (unlikely(vq->enabled == 0)) return 0; - count = reserve_avail_buf(vq, count, &res_start_idx, &res_end_idx); + avail_idx = *((volatile uint16_t *)&vq->avail->idx); + start_idx = vq->last_used_idx; + free_entries = avail_idx - start_idx; + count = RTE_MIN(count, free_entries); + count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST); if (count == 0) return 0; - LOG_DEBUG(VHOST_DATA, "(%d) res_start_idx %d | res_end_idx Index %d\n", - dev->vid, res_start_idx, res_end_idx); + LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n", + dev->vid, start_idx, start_idx + count); /* Retrieve all of the desc indexes first to avoid caching issues. */ - rte_prefetch0(&vq->avail->ring[res_start_idx & (vq->size - 1)]); + rte_prefetch0(&vq->avail->ring[start_idx & (vq->size - 1)]); for (i = 0; i < count; i++) { - used_idx = (res_start_idx + i) & (vq->size - 1); + used_idx = (start_idx + i) & (vq->size - 1); desc_indexes[i] = vq->avail->ring[used_idx]; vq->used->ring[used_idx].id = desc_indexes[i]; vq->used->ring[used_idx].len = pkts[i]->pkt_len + @@ -302,7 +271,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, err = copy_mbuf_to_desc(dev, vq, pkts[i], desc_idx); if (unlikely(err)) { - used_idx = (res_start_idx + i) & (vq->size - 1); + used_idx = (start_idx + i) & (vq->size - 1); vq->used->ring[used_idx].len = dev->vhost_hlen; vhost_log_used_vring(dev, vq, offsetof(struct vring_used, ring[used_idx]), @@ -315,12 +284,8 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, rte_smp_wmb(); - /* Wait until it's our turn to add our buffer to the used ring. */ - while (unlikely(vq->last_used_idx != res_start_idx)) - rte_pause(); - *(volatile uint16_t *)&vq->used->idx += count; - vq->last_used_idx = res_end_idx; + vq->last_used_idx += count; vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), sizeof(vq->used->idx)); @@ -367,40 +332,30 @@ fill_vec_buf(struct vhost_virtqueue *vq, uint32_t avail_idx, } /* - * As many data cores may want to access available buffers concurrently, - * they need to be reserved. - * * Returns -1 on fail, 0 on success */ static inline int reserve_avail_buf_mergeable(struct vhost_virtqueue *vq, uint32_t size, - uint16_t *start, uint16_t *end, - struct buf_vector *buf_vec) + uint16_t *end, struct buf_vector *buf_vec) { - uint16_t res_start_idx; - uint16_t res_cur_idx; + uint16_t cur_idx; uint16_t avail_idx; - uint32_t allocated; - uint32_t vec_idx; - uint16_t tries; + uint32_t allocated = 0; + uint32_t vec_idx = 0; + uint16_t tries = 0; -again: - res_start_idx = vq->last_used_idx_res; - res_cur_idx = res_start_idx; + cur_idx = vq->last_used_idx; - allocated = 0; - vec_idx = 0; - tries = 0; while (1) { avail_idx = *((volatile uint16_t *)&vq->avail->idx); - if (unlikely(res_cur_idx == avail_idx)) + if (unlikely(cur_idx == avail_idx)) return -1; - if (unlikely(fill_vec_buf(vq, res_cur_idx, &allocated, + if (unlikely(fill_vec_buf(vq, cur_idx, &allocated, &vec_idx, buf_vec) < 0)) return -1; - res_cur_idx++; + cur_idx++; tries++; if (allocated >= size) @@ -415,27 +370,19 @@ again: return -1; } - /* - * update vq->last_used_idx_res atomically. - * retry again if failed. - */ - if (rte_atomic16_cmpset(&vq->last_used_idx_res, - res_start_idx, res_cur_idx) == 0) - goto again; - - *start = res_start_idx; - *end = res_cur_idx; + *end = cur_idx; return 0; } static inline uint32_t __attribute__((always_inline)) copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint16_t res_start_idx, uint16_t res_end_idx, - struct rte_mbuf *m, struct buf_vector *buf_vec) + uint16_t end_idx, struct rte_mbuf *m, + struct buf_vector *buf_vec) { struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0}; uint32_t vec_idx = 0; - uint16_t cur_idx = res_start_idx; + uint16_t start_idx = vq->last_used_idx; + uint16_t cur_idx = start_idx; uint64_t desc_addr; uint32_t mbuf_offset, mbuf_avail; uint32_t desc_offset, desc_avail; @@ -446,15 +393,15 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n", - dev->vid, cur_idx, res_end_idx); - - if (buf_vec[vec_idx].buf_len < dev->vhost_hlen) - return -1; + dev->vid, cur_idx, end_idx); desc_addr = gpa_to_vva(dev, buf_vec[vec_idx].buf_addr); + if (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) + return 0; + rte_prefetch0((void *)(uintptr_t)desc_addr); - virtio_hdr.num_buffers = res_end_idx - res_start_idx; + virtio_hdr.num_buffers = end_idx - start_idx; LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n", dev->vid, virtio_hdr.num_buffers); @@ -486,6 +433,8 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, vec_idx++; desc_addr = gpa_to_vva(dev, buf_vec[vec_idx].buf_addr); + if (unlikely(!desc_addr)) + return 0; /* Prefetch buffer address. */ rte_prefetch0((void *)(uintptr_t)desc_addr); @@ -523,7 +472,7 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, offsetof(struct vring_used, ring[used_idx]), sizeof(vq->used->ring[used_idx])); - return res_end_idx - res_start_idx; + return end_idx - start_idx; } static inline uint32_t __attribute__((always_inline)) @@ -532,7 +481,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, { struct vhost_virtqueue *vq; uint32_t pkt_idx = 0, nr_used = 0; - uint16_t start, end; + uint16_t end; struct buf_vector buf_vec[BUF_VECTOR_MAX]; LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); @@ -553,7 +502,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, for (pkt_idx = 0; pkt_idx < count; pkt_idx++) { uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; - if (unlikely(reserve_avail_buf_mergeable(vq, pkt_len, &start, + if (unlikely(reserve_avail_buf_mergeable(vq, pkt_len, &end, buf_vec) < 0)) { LOG_DEBUG(VHOST_DATA, "(%d) failed to get enough desc from vring\n", @@ -561,21 +510,14 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, break; } - nr_used = copy_mbuf_to_desc_mergeable(dev, vq, start, end, + nr_used = copy_mbuf_to_desc_mergeable(dev, vq, end, pkts[pkt_idx], buf_vec); rte_smp_wmb(); - /* - * Wait until it's our turn to add our buffer - * to the used ring. - */ - while (unlikely(vq->last_used_idx != start)) - rte_pause(); - *(volatile uint16_t *)&vq->used->idx += nr_used; vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), sizeof(vq->used->idx)); - vq->last_used_idx = end; + vq->last_used_idx += nr_used; } if (likely(pkt_idx)) { @@ -756,24 +698,64 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, return -1; desc_addr = gpa_to_vva(dev, desc->addr); - rte_prefetch0((void *)(uintptr_t)desc_addr); + if (unlikely(!desc_addr)) + return -1; - /* Retrieve virtio net header */ hdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr); - desc_avail = desc->len - dev->vhost_hlen; - desc_offset = dev->vhost_hlen; + rte_prefetch0(hdr); + + /* + * A virtio driver normally uses at least 2 desc buffers + * for Tx: the first for storing the header, and others + * for storing the data. + */ + if (likely((desc->len == dev->vhost_hlen) && + (desc->flags & VRING_DESC_F_NEXT) != 0)) { + desc = &vq->desc[desc->next]; + + desc_addr = gpa_to_vva(dev, desc->addr); + if (unlikely(!desc_addr)) + return -1; + + rte_prefetch0((void *)(uintptr_t)desc_addr); + + desc_offset = 0; + desc_avail = desc->len; + nr_desc += 1; + + PRINT_PACKET(dev, (uintptr_t)desc_addr, desc->len, 0); + } else { + desc_avail = desc->len - dev->vhost_hlen; + desc_offset = dev->vhost_hlen; + } mbuf_offset = 0; mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM; - while (desc_avail != 0 || (desc->flags & VRING_DESC_F_NEXT) != 0) { + while (1) { + cpy_len = RTE_MIN(desc_avail, mbuf_avail); + rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *, mbuf_offset), + (void *)((uintptr_t)(desc_addr + desc_offset)), + cpy_len); + + mbuf_avail -= cpy_len; + mbuf_offset += cpy_len; + desc_avail -= cpy_len; + desc_offset += cpy_len; + /* This desc reaches to its end, get the next one */ if (desc_avail == 0) { + if ((desc->flags & VRING_DESC_F_NEXT) == 0) + break; + if (unlikely(desc->next >= vq->size || - ++nr_desc >= vq->size)) + ++nr_desc > vq->size)) return -1; desc = &vq->desc[desc->next]; desc_addr = gpa_to_vva(dev, desc->addr); + if (unlikely(!desc_addr)) + return -1; + rte_prefetch0((void *)(uintptr_t)desc_addr); desc_offset = 0; @@ -803,16 +785,6 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, mbuf_offset = 0; mbuf_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM; } - - cpy_len = RTE_MIN(desc_avail, mbuf_avail); - rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *, mbuf_offset), - (void *)((uintptr_t)(desc_addr + desc_offset)), - cpy_len); - - mbuf_avail -= cpy_len; - mbuf_offset += cpy_len; - desc_avail -= cpy_len; - desc_offset += cpy_len; } prev->data_len = mbuf_offset;