X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_rxtx.c;h=fa9c855e86ab73a90401c61d58d9756a9ee67db4;hb=6d13ea8e8e49ab957deae2bba5ecf4a4bfe747d1;hp=8564f18a7c6b861c9307d2056d30b96f9006cee2;hpb=a76290c8f1cf9c4774c23592921302a04a90bded;p=dpdk.git diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index 8564f18a7c..fa9c855e86 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -62,13 +62,13 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) struct vq_desc_extra *dxp; uint16_t desc_idx_last = desc_idx; - dp = &vq->vq_ring.desc[desc_idx]; + dp = &vq->vq_split.ring.desc[desc_idx]; dxp = &vq->vq_descx[desc_idx]; vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { while (dp->flags & VRING_DESC_F_NEXT) { desc_idx_last = dp->next; - dp = &vq->vq_ring.desc[dp->next]; + dp = &vq->vq_split.ring.desc[dp->next]; } } dxp->ndescs = 0; @@ -81,7 +81,7 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { vq->vq_desc_head_idx = desc_idx; } else { - dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx]; + dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; dp_tail->next = desc_idx; } @@ -118,12 +118,13 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, struct vring_packed_desc *desc; uint16_t i; - desc = vq->ring_packed.desc_packed; + desc = vq->vq_packed.ring.desc; for (i = 0; i < num; i++) { used_idx = vq->vq_used_cons_idx; if (!desc_is_used(&desc[used_idx], vq)) return i; + virtio_rmb(vq->hw->weak_barriers); len[i] = desc[used_idx].len; id = desc[used_idx].id; cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; @@ -140,7 +141,7 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, vq->vq_used_cons_idx++; if (vq->vq_used_cons_idx >= vq->vq_nentries) { vq->vq_used_cons_idx -= vq->vq_nentries; - vq->used_wrap_counter ^= 1; + vq->vq_packed.used_wrap_counter ^= 1; } } @@ -159,7 +160,7 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, /* Caller does the check */ for (i = 0; i < num ; i++) { used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); - uep = &vq->vq_ring.used->ring[used_idx]; + uep = &vq->vq_split.ring.used->ring[used_idx]; desc_idx = (uint16_t) uep->id; len[i] = uep->len; cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; @@ -198,7 +199,7 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq, for (i = 0; i < num; i++) { used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); /* Desc idx same as used idx */ - uep = &vq->vq_ring.used->ring[used_idx]; + uep = &vq->vq_split.ring.used->ring[used_idx]; len[i] = uep->len; cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; @@ -223,24 +224,55 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq, #define DEFAULT_TX_FREE_THRESH 32 #endif -/* Cleanup from completed transmits. */ static void -virtio_xmit_cleanup_packed(struct virtqueue *vq, int num) +virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num) +{ + uint16_t used_idx, id, curr_id, free_cnt = 0; + uint16_t size = vq->vq_nentries; + struct vring_packed_desc *desc = vq->vq_packed.ring.desc; + struct vq_desc_extra *dxp; + + used_idx = vq->vq_used_cons_idx; + while (num > 0 && desc_is_used(&desc[used_idx], vq)) { + virtio_rmb(vq->hw->weak_barriers); + id = desc[used_idx].id; + do { + curr_id = used_idx; + dxp = &vq->vq_descx[used_idx]; + used_idx += dxp->ndescs; + free_cnt += dxp->ndescs; + num -= dxp->ndescs; + if (used_idx >= size) { + used_idx -= size; + vq->vq_packed.used_wrap_counter ^= 1; + } + if (dxp->cookie != NULL) { + rte_pktmbuf_free(dxp->cookie); + dxp->cookie = NULL; + } + } while (curr_id != id); + } + vq->vq_used_cons_idx = used_idx; + vq->vq_free_cnt += free_cnt; +} + +static void +virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num) { uint16_t used_idx, id; uint16_t size = vq->vq_nentries; - struct vring_packed_desc *desc = vq->ring_packed.desc_packed; + struct vring_packed_desc *desc = vq->vq_packed.ring.desc; struct vq_desc_extra *dxp; used_idx = vq->vq_used_cons_idx; while (num-- && desc_is_used(&desc[used_idx], vq)) { - used_idx = vq->vq_used_cons_idx; + virtio_rmb(vq->hw->weak_barriers); id = desc[used_idx].id; dxp = &vq->vq_descx[id]; vq->vq_used_cons_idx += dxp->ndescs; if (vq->vq_used_cons_idx >= size) { vq->vq_used_cons_idx -= size; - vq->used_wrap_counter ^= 1; + vq->vq_packed.used_wrap_counter ^= 1; } vq_ring_free_id_packed(vq, id); if (dxp->cookie != NULL) { @@ -251,6 +283,16 @@ virtio_xmit_cleanup_packed(struct virtqueue *vq, int num) } } +/* Cleanup from completed transmits. */ +static inline void +virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order) +{ + if (in_order) + virtio_xmit_cleanup_inorder_packed(vq, num); + else + virtio_xmit_cleanup_normal_packed(vq, num); +} + static void virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num) { @@ -260,7 +302,7 @@ virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num) struct vq_desc_extra *dxp; used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); - uep = &vq->vq_ring.used->ring[used_idx]; + uep = &vq->vq_split.ring.used->ring[used_idx]; desc_idx = (uint16_t) uep->id; dxp = &vq->vq_descx[desc_idx]; @@ -278,7 +320,7 @@ virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num) static void virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num) { - uint16_t i, used_idx, desc_idx = 0, last_idx; + uint16_t i, idx = vq->vq_used_cons_idx; int16_t free_cnt = 0; struct vq_desc_extra *dxp = NULL; @@ -286,27 +328,16 @@ virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num) return; for (i = 0; i < num; i++) { - struct vring_used_elem *uep; - - used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); - uep = &vq->vq_ring.used->ring[used_idx]; - desc_idx = (uint16_t)uep->id; - - dxp = &vq->vq_descx[desc_idx]; - vq->vq_used_cons_idx++; - + dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)]; + free_cnt += dxp->ndescs; if (dxp->cookie != NULL) { rte_pktmbuf_free(dxp->cookie); dxp->cookie = NULL; } } - last_idx = desc_idx + dxp->ndescs - 1; - free_cnt = last_idx - vq->vq_desc_tail_idx; - if (free_cnt <= 0) - free_cnt += vq->vq_nentries; - - vq_ring_free_inorder(vq, last_idx, free_cnt); + vq->vq_free_cnt += free_cnt; + vq->vq_used_cons_idx = idx; } static inline int @@ -325,7 +356,7 @@ virtqueue_enqueue_refill_inorder(struct virtqueue *vq, return -EMSGSIZE; head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); - start_dp = vq->vq_ring.desc; + start_dp = vq->vq_split.ring.desc; while (i < num) { idx = head_idx & (vq->vq_nentries - 1); @@ -353,41 +384,44 @@ virtqueue_enqueue_refill_inorder(struct virtqueue *vq, } static inline int -virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) +virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, + uint16_t num) { struct vq_desc_extra *dxp; struct virtio_hw *hw = vq->hw; - struct vring_desc *start_dp; - uint16_t needed = 1; - uint16_t head_idx, idx; + struct vring_desc *start_dp = vq->vq_split.ring.desc; + uint16_t idx, i; if (unlikely(vq->vq_free_cnt == 0)) return -ENOSPC; - if (unlikely(vq->vq_free_cnt < needed)) + if (unlikely(vq->vq_free_cnt < num)) return -EMSGSIZE; - head_idx = vq->vq_desc_head_idx; - if (unlikely(head_idx >= vq->vq_nentries)) + if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) return -EFAULT; - idx = head_idx; - dxp = &vq->vq_descx[idx]; - dxp->cookie = (void *)cookie; - dxp->ndescs = needed; + for (i = 0; i < num; i++) { + idx = vq->vq_desc_head_idx; + dxp = &vq->vq_descx[idx]; + dxp->cookie = (void *)cookie[i]; + dxp->ndescs = 1; - start_dp = vq->vq_ring.desc; - start_dp[idx].addr = - VIRTIO_MBUF_ADDR(cookie, vq) + - RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; - start_dp[idx].len = - cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; - start_dp[idx].flags = VRING_DESC_F_WRITE; - idx = start_dp[idx].next; - vq->vq_desc_head_idx = idx; - if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = idx; - vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); - vq_update_avail_ring(vq, head_idx); + start_dp[idx].addr = + VIRTIO_MBUF_ADDR(cookie[i], vq) + + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; + start_dp[idx].len = + cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + + hw->vtnet_hdr_size; + start_dp[idx].flags = VRING_DESC_F_WRITE; + vq->vq_desc_head_idx = start_dp[idx].next; + vq_update_avail_ring(vq, idx); + if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { + vq->vq_desc_tail_idx = vq->vq_desc_head_idx; + break; + } + } + + vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); return 0; } @@ -396,8 +430,8 @@ static inline int virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, struct rte_mbuf **cookie, uint16_t num) { - struct vring_packed_desc *start_dp = vq->ring_packed.desc_packed; - uint16_t flags = VRING_DESC_F_WRITE | vq->avail_used_flags; + struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; + uint16_t flags = vq->vq_packed.cached_flags; struct virtio_hw *hw = vq->hw; struct vq_desc_extra *dxp; uint16_t idx; @@ -422,15 +456,13 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, vq->vq_desc_head_idx = dxp->next; if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) vq->vq_desc_tail_idx = vq->vq_desc_head_idx; - rte_smp_wmb(); + virtio_wmb(hw->weak_barriers); start_dp[idx].flags = flags; if (++vq->vq_avail_idx >= vq->vq_nentries) { vq->vq_avail_idx -= vq->vq_nentries; - vq->avail_wrap_counter ^= 1; - vq->avail_used_flags = - VRING_DESC_F_AVAIL(vq->avail_wrap_counter) | - VRING_DESC_F_USED(!vq->avail_wrap_counter); - flags = VRING_DESC_F_WRITE | vq->avail_used_flags; + vq->vq_packed.cached_flags ^= + VRING_PACKED_DESC_F_AVAIL_USED; + flags = vq->vq_packed.cached_flags; } } vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); @@ -485,6 +517,15 @@ virtio_tso_fix_cksum(struct rte_mbuf *m) (var) = (val); \ } while (0) +#define virtqueue_clear_net_hdr(_hdr) do { \ + ASSIGN_UNLESS_EQUAL((_hdr)->csum_start, 0); \ + ASSIGN_UNLESS_EQUAL((_hdr)->csum_offset, 0); \ + ASSIGN_UNLESS_EQUAL((_hdr)->flags, 0); \ + ASSIGN_UNLESS_EQUAL((_hdr)->gso_type, 0); \ + ASSIGN_UNLESS_EQUAL((_hdr)->gso_size, 0); \ + ASSIGN_UNLESS_EQUAL((_hdr)->hdr_len, 0); \ +} while (0) + static inline void virtqueue_xmit_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *cookie, @@ -548,11 +589,11 @@ virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, uint16_t i = 0; idx = vq->vq_desc_head_idx; - start_dp = vq->vq_ring.desc; + start_dp = vq->vq_split.ring.desc; while (i < num) { idx = idx & (vq->vq_nentries - 1); - dxp = &vq->vq_descx[idx]; + dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; dxp->cookie = (void *)cookies[i]; dxp->ndescs = 1; @@ -560,18 +601,11 @@ virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, rte_pktmbuf_prepend(cookies[i], head_size); cookies[i]->pkt_len -= head_size; - /* if offload disabled, it is not zeroed below, do it now */ - if (!vq->hw->has_tx_offload) { - ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); - ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); - ASSIGN_UNLESS_EQUAL(hdr->flags, 0); - ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); - ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); - ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); - } - - virtqueue_xmit_offload(hdr, cookies[i], - vq->hw->has_tx_offload); + /* if offload disabled, hdr is not zeroed yet, do it now */ + if (!vq->hw->has_tx_offload) + virtqueue_clear_net_hdr(hdr); + else + virtqueue_xmit_offload(hdr, cookies[i], true); start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq); start_dp[idx].len = cookies[i]->data_len; @@ -587,9 +621,63 @@ virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); } +static inline void +virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, + struct rte_mbuf *cookie, + int in_order) +{ + struct virtqueue *vq = txvq->vq; + struct vring_packed_desc *dp; + struct vq_desc_extra *dxp; + uint16_t idx, id, flags; + uint16_t head_size = vq->hw->vtnet_hdr_size; + struct virtio_net_hdr *hdr; + + id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; + idx = vq->vq_avail_idx; + dp = &vq->vq_packed.ring.desc[idx]; + + dxp = &vq->vq_descx[id]; + dxp->ndescs = 1; + dxp->cookie = cookie; + + flags = vq->vq_packed.cached_flags; + + /* prepend cannot fail, checked by caller */ + hdr = (struct virtio_net_hdr *) + rte_pktmbuf_prepend(cookie, head_size); + cookie->pkt_len -= head_size; + + /* if offload disabled, hdr is not zeroed yet, do it now */ + if (!vq->hw->has_tx_offload) + virtqueue_clear_net_hdr(hdr); + else + virtqueue_xmit_offload(hdr, cookie, true); + + dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); + dp->len = cookie->data_len; + dp->id = id; + + if (++vq->vq_avail_idx >= vq->vq_nentries) { + vq->vq_avail_idx -= vq->vq_nentries; + vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; + } + + vq->vq_free_cnt--; + + if (!in_order) { + vq->vq_desc_head_idx = dxp->next; + if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) + vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; + } + + virtio_wmb(vq->hw->weak_barriers); + dp->flags = flags; +} + static inline void virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, - uint16_t needed, int can_push) + uint16_t needed, int can_push, int in_order) { struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; struct vq_desc_extra *dxp; @@ -600,7 +688,7 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, struct virtio_net_hdr *hdr; uint16_t prev; - id = vq->vq_desc_head_idx; + id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; dxp = &vq->vq_descx[id]; dxp->ndescs = needed; @@ -609,11 +697,11 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, head_idx = vq->vq_avail_idx; idx = head_idx; prev = head_idx; - start_dp = vq->ring_packed.desc_packed; + start_dp = vq->vq_packed.ring.desc; - head_dp = &vq->ring_packed.desc_packed[idx]; + head_dp = &vq->vq_packed.ring.desc[idx]; head_flags = cookie->next ? VRING_DESC_F_NEXT : 0; - head_flags |= vq->avail_used_flags; + head_flags |= vq->vq_packed.cached_flags; if (can_push) { /* prepend cannot fail, checked by caller */ @@ -625,14 +713,8 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, cookie->pkt_len -= head_size; /* if offload disabled, it is not zeroed below, do it now */ - if (!vq->hw->has_tx_offload) { - ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); - ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); - ASSIGN_UNLESS_EQUAL(hdr->flags, 0); - ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); - ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); - ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); - } + if (!vq->hw->has_tx_offload) + virtqueue_clear_net_hdr(hdr); } else { /* setup first tx ring slot to point to header * stored in reserved region. @@ -644,10 +726,8 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, idx++; if (idx >= vq->vq_nentries) { idx -= vq->vq_nentries; - vq->avail_wrap_counter ^= 1; - vq->avail_used_flags = - VRING_DESC_F_AVAIL(vq->avail_wrap_counter) | - VRING_DESC_F_USED(!vq->avail_wrap_counter); + vq->vq_packed.cached_flags ^= + VRING_PACKED_DESC_F_AVAIL_USED; } } @@ -660,31 +740,30 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, start_dp[idx].len = cookie->data_len; if (likely(idx != head_idx)) { flags = cookie->next ? VRING_DESC_F_NEXT : 0; - flags |= vq->avail_used_flags; + flags |= vq->vq_packed.cached_flags; start_dp[idx].flags = flags; } prev = idx; idx++; if (idx >= vq->vq_nentries) { idx -= vq->vq_nentries; - vq->avail_wrap_counter ^= 1; - vq->avail_used_flags = - VRING_DESC_F_AVAIL(vq->avail_wrap_counter) | - VRING_DESC_F_USED(!vq->avail_wrap_counter); + vq->vq_packed.cached_flags ^= + VRING_PACKED_DESC_F_AVAIL_USED; } } while ((cookie = cookie->next) != NULL); start_dp[prev].id = id; vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); - - vq->vq_desc_head_idx = dxp->next; - if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; - vq->vq_avail_idx = idx; - rte_smp_wmb(); + if (!in_order) { + vq->vq_desc_head_idx = dxp->next; + if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) + vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; + } + + virtio_wmb(vq->hw->weak_barriers); head_dp->flags = head_flags; } @@ -704,11 +783,14 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, head_idx = vq->vq_desc_head_idx; idx = head_idx; - dxp = &vq->vq_descx[idx]; + if (in_order) + dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; + else + dxp = &vq->vq_descx[idx]; dxp->cookie = (void *)cookie; dxp->ndescs = needed; - start_dp = vq->vq_ring.desc; + start_dp = vq->vq_split.ring.desc; if (can_push) { /* prepend cannot fail, checked by caller */ @@ -720,14 +802,8 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, cookie->pkt_len -= head_size; /* if offload disabled, it is not zeroed below, do it now */ - if (!vq->hw->has_tx_offload) { - ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); - ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); - ASSIGN_UNLESS_EQUAL(hdr->flags, 0); - ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); - ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); - ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); - } + if (!vq->hw->has_tx_offload) + virtqueue_clear_net_hdr(hdr); } else if (use_indirect) { /* setup tx ring slot to point to indirect * descriptor list stored in reserved region. @@ -767,7 +843,7 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, } while ((cookie = cookie->next) != NULL); if (use_indirect) - idx = vq->vq_ring.desc[head_idx].next; + idx = vq->vq_split.ring.desc[head_idx].next; vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); @@ -842,8 +918,8 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) if (hw->use_simple_rx) { for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) { - vq->vq_ring.avail->ring[desc_idx] = desc_idx; - vq->vq_ring.desc[desc_idx].flags = + vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; + vq->vq_split.ring.desc[desc_idx].flags = VRING_DESC_F_WRITE; } @@ -892,7 +968,8 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); else - error = virtqueue_enqueue_recv_refill(vq, m); + error = virtqueue_enqueue_recv_refill(vq, + &m, 1); if (error) { rte_pktmbuf_free(m); break; @@ -972,7 +1049,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, if (!vtpci_packed_queue(hw)) { if (hw->use_inorder_tx) - vq->vq_ring.desc[vq->vq_nentries - 1].next = 0; + vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; } VIRTQUEUE_DUMP(vq); @@ -980,7 +1057,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, return 0; } -static void +static inline void virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) { int error; @@ -991,7 +1068,7 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) if (vtpci_packed_queue(vq->hw)) error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); else - error = virtqueue_enqueue_recv_refill(vq, m); + error = virtqueue_enqueue_recv_refill(vq, &m, 1); if (unlikely(error)) { RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf"); @@ -999,7 +1076,7 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) } } -static void +static inline void virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) { int error; @@ -1011,11 +1088,11 @@ virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) } } -static void +static inline void virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) { uint32_t s = mbuf->pkt_len; - struct ether_addr *ea; + struct rte_ether_addr *ea; stats->bytes += s; @@ -1032,11 +1109,11 @@ virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) stats->size_bins[0]++; else if (s < 1519) stats->size_bins[6]++; - else if (s >= 1519) + else stats->size_bins[7]++; } - ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *); + ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); if (is_multicast_ether_addr(ea)) { if (is_broadcast_ether_addr(ea)) stats->broadcast++; @@ -1054,7 +1131,7 @@ virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) } /* Optionally fill offload information in structure */ -static int +static inline int virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) { struct rte_net_hdr_lens hdr_lens; @@ -1133,7 +1210,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) struct virtnet_rx *rxvq = rx_queue; struct virtqueue *vq = rxvq->vq; struct virtio_hw *hw = vq->hw; - struct rte_mbuf *rxm, *new_mbuf; + struct rte_mbuf *rxm; uint16_t nb_used, num, nb_rx; uint32_t len[VIRTIO_MBUF_BURST_SZ]; struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; @@ -1148,7 +1225,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(); + virtio_rmb(hw->weak_barriers); num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) @@ -1203,20 +1280,24 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxvq->stats.packets += nb_rx; /* Allocate new mbuf for the used descriptor */ - while (likely(!virtqueue_full(vq))) { - new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool); - if (unlikely(new_mbuf == NULL)) { - struct rte_eth_dev *dev - = &rte_eth_devices[rxvq->port_id]; - dev->data->rx_mbuf_alloc_failed++; - break; - } - error = virtqueue_enqueue_recv_refill(vq, new_mbuf); - if (unlikely(error)) { - rte_pktmbuf_free(new_mbuf); - break; + if (likely(!virtqueue_full(vq))) { + uint16_t free_cnt = vq->vq_free_cnt; + struct rte_mbuf *new_pkts[free_cnt]; + + if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, + free_cnt) == 0)) { + error = virtqueue_enqueue_recv_refill(vq, new_pkts, + free_cnt); + if (unlikely(error)) { + for (i = 0; i < free_cnt; i++) + rte_pktmbuf_free(new_pkts[i]); + } + nb_enqueued += free_cnt; + } else { + struct rte_eth_dev *dev = + &rte_eth_devices[rxvq->port_id]; + dev->data->rx_mbuf_alloc_failed += free_cnt; } - nb_enqueued++; } if (likely(nb_enqueued)) { @@ -1238,7 +1319,7 @@ virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, struct virtnet_rx *rxvq = rx_queue; struct virtqueue *vq = rxvq->vq; struct virtio_hw *hw = vq->hw; - struct rte_mbuf *rxm, *new_mbuf; + struct rte_mbuf *rxm; uint16_t num, nb_rx; uint32_t len[VIRTIO_MBUF_BURST_SZ]; struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; @@ -1302,20 +1383,24 @@ virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, rxvq->stats.packets += nb_rx; /* Allocate new mbuf for the used descriptor */ - while (likely(!virtqueue_full(vq))) { - new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool); - if (unlikely(new_mbuf == NULL)) { + if (likely(!virtqueue_full(vq))) { + uint16_t free_cnt = vq->vq_free_cnt; + struct rte_mbuf *new_pkts[free_cnt]; + + if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, + free_cnt) == 0)) { + error = virtqueue_enqueue_recv_refill_packed(vq, + new_pkts, free_cnt); + if (unlikely(error)) { + for (i = 0; i < free_cnt; i++) + rte_pktmbuf_free(new_pkts[i]); + } + nb_enqueued += free_cnt; + } else { struct rte_eth_dev *dev = &rte_eth_devices[rxvq->port_id]; - dev->data->rx_mbuf_alloc_failed++; - break; - } - error = virtqueue_enqueue_recv_refill_packed(vq, &new_mbuf, 1); - if (unlikely(error)) { - rte_pktmbuf_free(new_mbuf); - break; + dev->data->rx_mbuf_alloc_failed += free_cnt; } - nb_enqueued++; } if (likely(nb_enqueued)) { @@ -1330,7 +1415,7 @@ virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t -virtio_recv_mergeable_pkts_inorder(void *rx_queue, +virtio_recv_pkts_inorder(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -1357,7 +1442,7 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue, nb_used = RTE_MIN(nb_used, nb_pkts); nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); - virtio_rmb(); + virtio_rmb(hw->weak_barriers); PMD_RX_LOG(DEBUG, "used:%d", nb_used); @@ -1387,10 +1472,14 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue, header = (struct virtio_net_hdr_mrg_rxbuf *) ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); - seg_num = header->num_buffers; - if (seg_num == 0) + if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + seg_num = header->num_buffers; + if (seg_num == 0) + seg_num = 1; + } else { seg_num = 1; + } rxm->data_off = RTE_PKTMBUF_HEADROOM; rxm->nb_segs = seg_num; @@ -1448,6 +1537,7 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue, prev = rcv_pkts[nb_rx]; if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { + virtio_rmb(hw->weak_barriers); num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, rcv_cnt); uint16_t extra_idx = 0; @@ -1524,53 +1614,46 @@ virtio_recv_mergeable_pkts(void *rx_queue, struct virtnet_rx *rxvq = rx_queue; struct virtqueue *vq = rxvq->vq; struct virtio_hw *hw = vq->hw; - struct rte_mbuf *rxm, *new_mbuf; - uint16_t nb_used, num, nb_rx; + struct rte_mbuf *rxm; + struct rte_mbuf *prev; + uint16_t nb_used, num, nb_rx = 0; uint32_t len[VIRTIO_MBUF_BURST_SZ]; struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; - struct rte_mbuf *prev; int error; - uint32_t i, nb_enqueued; - uint32_t seg_num; - uint16_t extra_idx; - uint32_t seg_res; - uint32_t hdr_size; + uint32_t nb_enqueued = 0; + uint32_t seg_num = 0; + uint32_t seg_res = 0; + uint32_t hdr_size = hw->vtnet_hdr_size; + int32_t i; - nb_rx = 0; if (unlikely(hw->started == 0)) return nb_rx; nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(); + virtio_rmb(hw->weak_barriers); PMD_RX_LOG(DEBUG, "used:%d", nb_used); - i = 0; - nb_enqueued = 0; - seg_num = 0; - extra_idx = 0; - seg_res = 0; - hdr_size = hw->vtnet_hdr_size; - - while (i < nb_used) { - struct virtio_net_hdr_mrg_rxbuf *header; + num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; + if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) + num = VIRTIO_MBUF_BURST_SZ; + if (likely(num > DESC_PER_CACHELINE)) + num = num - ((vq->vq_used_cons_idx + num) % + DESC_PER_CACHELINE); - if (nb_rx == nb_pkts) - break; - num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1); - if (num != 1) - continue; + num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); - i++; + for (i = 0; i < num; i++) { + struct virtio_net_hdr_mrg_rxbuf *header; PMD_RX_LOG(DEBUG, "dequeue:%d", num); - PMD_RX_LOG(DEBUG, "packet len:%d", len[0]); + PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); - rxm = rcv_pkts[0]; + rxm = rcv_pkts[i]; - if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) { + if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) { PMD_RX_LOG(ERR, "Packet drop"); nb_enqueued++; virtio_discard_rxbuf(vq, rxm); @@ -1578,10 +1661,10 @@ virtio_recv_mergeable_pkts(void *rx_queue, continue; } - header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr + - RTE_PKTMBUF_HEADROOM - hdr_size); + header = (struct virtio_net_hdr_mrg_rxbuf *) + ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM + - hdr_size); seg_num = header->num_buffers; - if (seg_num == 0) seg_num = 1; @@ -1589,10 +1672,11 @@ virtio_recv_mergeable_pkts(void *rx_queue, rxm->nb_segs = seg_num; rxm->ol_flags = 0; rxm->vlan_tci = 0; - rxm->pkt_len = (uint32_t)(len[0] - hdr_size); - rxm->data_len = (uint16_t)(len[0] - hdr_size); + rxm->pkt_len = (uint32_t)(len[i] - hdr_size); + rxm->data_len = (uint16_t)(len[i] - hdr_size); rxm->port = rxvq->port_id; + rx_pkts[nb_rx] = rxm; prev = rxm; @@ -1603,75 +1687,97 @@ virtio_recv_mergeable_pkts(void *rx_queue, continue; } + if (hw->vlan_strip) + rte_vlan_strip(rx_pkts[nb_rx]); + seg_res = seg_num - 1; - while (seg_res != 0) { - /* - * Get extra segments for current uncompleted packet. - */ - uint16_t rcv_cnt = - RTE_MIN(seg_res, RTE_DIM(rcv_pkts)); - if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { - uint32_t rx_num = - virtqueue_dequeue_burst_rx(vq, - rcv_pkts, len, rcv_cnt); - i += rx_num; - rcv_cnt = rx_num; - } else { - PMD_RX_LOG(ERR, - "No enough segments for packet."); - nb_enqueued++; - virtio_discard_rxbuf(vq, rxm); - rxvq->stats.errors++; - break; - } + /* Merge remaining segments */ + while (seg_res != 0 && i < (num - 1)) { + i++; + + rxm = rcv_pkts[i]; + rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; + rxm->pkt_len = (uint32_t)(len[i]); + rxm->data_len = (uint16_t)(len[i]); + + rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); + rx_pkts[nb_rx]->data_len += (uint16_t)(len[i]); + + if (prev) + prev->next = rxm; + + prev = rxm; + seg_res -= 1; + } + + if (!seg_res) { + virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); + nb_rx++; + } + } + + /* Last packet still need merge segments */ + while (seg_res != 0) { + uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, + VIRTIO_MBUF_BURST_SZ); - extra_idx = 0; + prev = rcv_pkts[nb_rx]; + if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { + virtio_rmb(hw->weak_barriers); + num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, + rcv_cnt); + uint16_t extra_idx = 0; + rcv_cnt = num; while (extra_idx < rcv_cnt) { rxm = rcv_pkts[extra_idx]; - - rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; + rxm->data_off = + RTE_PKTMBUF_HEADROOM - hdr_size; rxm->pkt_len = (uint32_t)(len[extra_idx]); rxm->data_len = (uint16_t)(len[extra_idx]); - - if (prev) - prev->next = rxm; - + prev->next = rxm; prev = rxm; - rx_pkts[nb_rx]->pkt_len += rxm->pkt_len; - extra_idx++; + rx_pkts[nb_rx]->pkt_len += len[extra_idx]; + rx_pkts[nb_rx]->data_len += len[extra_idx]; + extra_idx += 1; }; seg_res -= rcv_cnt; - } - - if (hw->vlan_strip) - rte_vlan_strip(rx_pkts[nb_rx]); - - VIRTIO_DUMP_PACKET(rx_pkts[nb_rx], - rx_pkts[nb_rx]->data_len); - virtio_update_packet_stats(&rxvq->stats, rx_pkts[nb_rx]); - nb_rx++; + if (!seg_res) { + virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); + nb_rx++; + } + } else { + PMD_RX_LOG(ERR, + "No enough segments for packet."); + virtio_discard_rxbuf(vq, prev); + rxvq->stats.errors++; + break; + } } rxvq->stats.packets += nb_rx; /* Allocate new mbuf for the used descriptor */ - while (likely(!virtqueue_full(vq))) { - new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool); - if (unlikely(new_mbuf == NULL)) { - struct rte_eth_dev *dev - = &rte_eth_devices[rxvq->port_id]; - dev->data->rx_mbuf_alloc_failed++; - break; - } - error = virtqueue_enqueue_recv_refill(vq, new_mbuf); - if (unlikely(error)) { - rte_pktmbuf_free(new_mbuf); - break; + if (likely(!virtqueue_full(vq))) { + /* free_cnt may include mrg descs */ + uint16_t free_cnt = vq->vq_free_cnt; + struct rte_mbuf *new_pkts[free_cnt]; + + if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { + error = virtqueue_enqueue_recv_refill(vq, new_pkts, + free_cnt); + if (unlikely(error)) { + for (i = 0; i < free_cnt; i++) + rte_pktmbuf_free(new_pkts[i]); + } + nb_enqueued += free_cnt; + } else { + struct rte_eth_dev *dev = + &rte_eth_devices[rxvq->port_id]; + dev->data->rx_mbuf_alloc_failed += free_cnt; } - nb_enqueued++; } if (likely(nb_enqueued)) { @@ -1871,6 +1977,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, struct virtio_hw *hw = vq->hw; uint16_t hdr_size = hw->vtnet_hdr_size; uint16_t nb_tx = 0; + bool in_order = hw->use_inorder_tx; int error; if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) @@ -1882,7 +1989,8 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); if (nb_pkts > vq->vq_free_cnt) - virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt); + virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, + in_order); for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { struct rte_mbuf *txm = tx_pkts[nb_tx]; @@ -1895,6 +2003,8 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, rte_pktmbuf_free(txm); continue; } + /* vlan_insert may add a header mbuf */ + tx_pkts[nb_tx] = txm; } /* optimize ring usage */ @@ -1917,9 +2027,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, /* Positive value indicates it need free vring descriptors */ if (unlikely(need > 0)) { - virtio_rmb(); - need = RTE_MIN(need, (int)nb_pkts); - virtio_xmit_cleanup_packed(vq, need); + virtio_xmit_cleanup_packed(vq, need, in_order); need = slots - vq->vq_free_cnt; if (unlikely(need > 0)) { PMD_TX_LOG(ERR, @@ -1929,9 +2037,12 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, } /* Enqueue Packet buffers */ - virtqueue_enqueue_xmit_packed(txvq, txm, slots, can_push); + if (can_push) + virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); + else + virtqueue_enqueue_xmit_packed(txvq, txm, slots, 0, + in_order); - txvq->stats.bytes += txm->pkt_len; virtio_update_packet_stats(&txvq->stats, txm); } @@ -1966,7 +2077,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(); + virtio_rmb(hw->weak_barriers); if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) virtio_xmit_cleanup(vq, nb_used); @@ -1981,6 +2092,8 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) rte_pktmbuf_free(txm); continue; } + /* vlan_insert may add a header mbuf */ + tx_pkts[nb_tx] = txm; } /* optimize ring usage */ @@ -2008,7 +2121,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* Positive value indicates it need free vring descriptors */ if (unlikely(need > 0)) { nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(); + virtio_rmb(hw->weak_barriers); need = RTE_MIN(need, (int)nb_used); virtio_xmit_cleanup(vq, need); @@ -2064,7 +2177,7 @@ virtio_xmit_pkts_inorder(void *tx_queue, PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(); + virtio_rmb(hw->weak_barriers); if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) virtio_xmit_cleanup_inorder(vq, nb_used); @@ -2084,6 +2197,8 @@ virtio_xmit_pkts_inorder(void *tx_queue, rte_pktmbuf_free(txm); continue; } + /* vlan_insert may add a header mbuf */ + tx_pkts[nb_tx] = txm; } /* optimize ring usage */ @@ -2112,7 +2227,7 @@ virtio_xmit_pkts_inorder(void *tx_queue, need = slots - vq->vq_free_cnt; if (unlikely(need > 0)) { nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(); + virtio_rmb(hw->weak_barriers); need = RTE_MIN(need, (int)nb_used); virtio_xmit_cleanup_inorder(vq, need);