X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_rxtx.c;h=4795893ec7fae49487d285641ba50c70329c9634;hb=5729407fa4d612b93ceefacef13f57c1de2ceaa6;hp=df0fe066f93457812edcaa2a65dc8da83bb7e21d;hpb=1cbc164ebd8a11a5e850454d7425f01612a74bc4;p=dpdk.git diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index df0fe066f9..4795893ec7 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -27,7 +27,7 @@ #include "virtio_logs.h" #include "virtio_ethdev.h" -#include "virtio_pci.h" +#include "virtio.h" #include "virtqueue.h" #include "virtio_rxtx.h" #include "virtio_rxtx_simple.h" @@ -39,15 +39,6 @@ #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) #endif -int -virtio_dev_rx_queue_done(void *rxq, uint16_t offset) -{ - struct virtnet_rx *rxvq = rxq; - struct virtqueue *vq = rxvq->vq; - - return VIRTQUEUE_NUSED(vq) >= offset; -} - void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) { @@ -89,21 +80,46 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) dp->next = VQ_RING_DESC_CHAIN_END; } -static void -vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id) +void +virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) { - struct vq_desc_extra *dxp; + uint32_t s = mbuf->pkt_len; + struct rte_ether_addr *ea; - dxp = &vq->vq_descx[id]; - vq->vq_free_cnt += dxp->ndescs; + stats->bytes += s; - if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) - vq->vq_desc_head_idx = id; - else - vq->vq_descx[vq->vq_desc_tail_idx].next = id; + if (s == 64) { + stats->size_bins[1]++; + } else if (s > 64 && s < 1024) { + uint32_t bin; + + /* count zeros, and offset into correct bin */ + bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; + stats->size_bins[bin]++; + } else { + if (s < 64) + stats->size_bins[0]++; + else if (s < 1519) + stats->size_bins[6]++; + else + stats->size_bins[7]++; + } - vq->vq_desc_tail_idx = id; - dxp->next = VQ_RING_DESC_CHAIN_END; + ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); + if (rte_is_multicast_ether_addr(ea)) { + if (rte_is_broadcast_ether_addr(ea)) + stats->broadcast++; + else + stats->multicast++; + } +} + +static inline void +virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) +{ + VIRTIO_DUMP_PACKET(m, m->data_len); + + virtio_update_packet_stats(&rxvq->stats, m); } static uint16_t @@ -122,9 +138,11 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, for (i = 0; i < num; i++) { used_idx = vq->vq_used_cons_idx; + /* desc_is_used has a load-acquire or rte_io_rmb inside + * and wait for used desc in virtqueue. + */ if (!desc_is_used(&desc[used_idx], vq)) return i; - virtio_rmb(vq->hw->weak_barriers); len[i] = desc[used_idx].len; id = desc[used_idx].id; cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; @@ -220,126 +238,6 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq, return i; } -#ifndef DEFAULT_TX_FREE_THRESH -#define DEFAULT_TX_FREE_THRESH 32 -#endif - -static void -virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num) -{ - uint16_t used_idx, id, curr_id, free_cnt = 0; - uint16_t size = vq->vq_nentries; - struct vring_packed_desc *desc = vq->vq_packed.ring.desc; - struct vq_desc_extra *dxp; - - used_idx = vq->vq_used_cons_idx; - while (num > 0 && desc_is_used(&desc[used_idx], vq)) { - virtio_rmb(vq->hw->weak_barriers); - id = desc[used_idx].id; - do { - curr_id = used_idx; - dxp = &vq->vq_descx[used_idx]; - used_idx += dxp->ndescs; - free_cnt += dxp->ndescs; - num -= dxp->ndescs; - if (used_idx >= size) { - used_idx -= size; - vq->vq_packed.used_wrap_counter ^= 1; - } - if (dxp->cookie != NULL) { - rte_pktmbuf_free(dxp->cookie); - dxp->cookie = NULL; - } - } while (curr_id != id); - } - vq->vq_used_cons_idx = used_idx; - vq->vq_free_cnt += free_cnt; -} - -static void -virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num) -{ - uint16_t used_idx, id; - uint16_t size = vq->vq_nentries; - struct vring_packed_desc *desc = vq->vq_packed.ring.desc; - struct vq_desc_extra *dxp; - - used_idx = vq->vq_used_cons_idx; - while (num-- && desc_is_used(&desc[used_idx], vq)) { - virtio_rmb(vq->hw->weak_barriers); - id = desc[used_idx].id; - dxp = &vq->vq_descx[id]; - vq->vq_used_cons_idx += dxp->ndescs; - if (vq->vq_used_cons_idx >= size) { - vq->vq_used_cons_idx -= size; - vq->vq_packed.used_wrap_counter ^= 1; - } - vq_ring_free_id_packed(vq, id); - if (dxp->cookie != NULL) { - rte_pktmbuf_free(dxp->cookie); - dxp->cookie = NULL; - } - used_idx = vq->vq_used_cons_idx; - } -} - -/* Cleanup from completed transmits. */ -static inline void -virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order) -{ - if (in_order) - virtio_xmit_cleanup_inorder_packed(vq, num); - else - virtio_xmit_cleanup_normal_packed(vq, num); -} - -static void -virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num) -{ - uint16_t i, used_idx, desc_idx; - for (i = 0; i < num; i++) { - struct vring_used_elem *uep; - struct vq_desc_extra *dxp; - - used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); - uep = &vq->vq_split.ring.used->ring[used_idx]; - - desc_idx = (uint16_t) uep->id; - dxp = &vq->vq_descx[desc_idx]; - vq->vq_used_cons_idx++; - vq_ring_free_chain(vq, desc_idx); - - if (dxp->cookie != NULL) { - rte_pktmbuf_free(dxp->cookie); - dxp->cookie = NULL; - } - } -} - -/* Cleanup from completed inorder transmits. */ -static void -virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num) -{ - uint16_t i, idx = vq->vq_used_cons_idx; - int16_t free_cnt = 0; - struct vq_desc_extra *dxp = NULL; - - if (unlikely(num == 0)) - return; - - for (i = 0; i < num; i++) { - dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)]; - free_cnt += dxp->ndescs; - if (dxp->cookie != NULL) { - rte_pktmbuf_free(dxp->cookie); - dxp->cookie = NULL; - } - } - - vq->vq_free_cnt += free_cnt; - vq->vq_used_cons_idx = idx; -} - static inline int virtqueue_enqueue_refill_inorder(struct virtqueue *vq, struct rte_mbuf **cookies, @@ -364,13 +262,10 @@ virtqueue_enqueue_refill_inorder(struct virtqueue *vq, dxp->cookie = (void *)cookies[i]; dxp->ndescs = 1; - start_dp[idx].addr = - VIRTIO_MBUF_ADDR(cookies[i], vq) + - RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; - start_dp[idx].len = - cookies[i]->buf_len - - RTE_PKTMBUF_HEADROOM + - hw->vtnet_hdr_size; + start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookies[i], vq) + + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; + start_dp[idx].len = cookies[i]->buf_len - + RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; start_dp[idx].flags = VRING_DESC_F_WRITE; vq_update_avail_ring(vq, idx); @@ -406,11 +301,9 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, dxp->cookie = (void *)cookie[i]; dxp->ndescs = 1; - start_dp[idx].addr = - VIRTIO_MBUF_ADDR(cookie[i], vq) + + start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; - start_dp[idx].len = - cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + + start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; start_dp[idx].flags = VRING_DESC_F_WRITE; vq->vq_desc_head_idx = start_dp[idx].next; @@ -426,13 +319,32 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, return 0; } +static inline void +virtqueue_refill_single_packed(struct virtqueue *vq, + struct vring_packed_desc *dp, + struct rte_mbuf *cookie) +{ + uint16_t flags = vq->vq_packed.cached_flags; + struct virtio_hw *hw = vq->hw; + + dp->addr = VIRTIO_MBUF_ADDR(cookie, vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; + dp->len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; + + virtqueue_store_flags_packed(dp, flags, hw->weak_barriers); + + if (++vq->vq_avail_idx >= vq->vq_nentries) { + vq->vq_avail_idx -= vq->vq_nentries; + vq->vq_packed.cached_flags ^= + VRING_PACKED_DESC_F_AVAIL_USED; + flags = vq->vq_packed.cached_flags; + } +} + static inline int -virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, +virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq, struct rte_mbuf **cookie, uint16_t num) { struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; - uint16_t flags = vq->vq_packed.cached_flags; - struct virtio_hw *hw = vq->hw; struct vq_desc_extra *dxp; uint16_t idx; int i; @@ -448,22 +360,34 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, dxp->cookie = (void *)cookie[i]; dxp->ndescs = 1; - start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) + - RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; - start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM - + hw->vtnet_hdr_size; + virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); + } + vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); + return 0; +} - vq->vq_desc_head_idx = dxp->next; - if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = vq->vq_desc_head_idx; - virtio_wmb(hw->weak_barriers); - start_dp[idx].flags = flags; - if (++vq->vq_avail_idx >= vq->vq_nentries) { - vq->vq_avail_idx -= vq->vq_nentries; - vq->vq_packed.cached_flags ^= - VRING_PACKED_DESC_F_AVAIL_USED; - flags = vq->vq_packed.cached_flags; - } +static inline int +virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, + struct rte_mbuf **cookie, uint16_t num) +{ + struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; + struct vq_desc_extra *dxp; + uint16_t idx, did; + int i; + + if (unlikely(vq->vq_free_cnt == 0)) + return -ENOSPC; + if (unlikely(vq->vq_free_cnt < num)) + return -EMSGSIZE; + + for (i = 0; i < num; i++) { + idx = vq->vq_avail_idx; + did = start_dp[idx].id; + dxp = &vq->vq_descx[did]; + dxp->cookie = (void *)cookie[i]; + dxp->ndescs = 1; + + virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); } vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); return 0; @@ -512,68 +436,7 @@ virtio_tso_fix_cksum(struct rte_mbuf *m) } -/* avoid write operation when necessary, to lessen cache issues */ -#define ASSIGN_UNLESS_EQUAL(var, val) do { \ - if ((var) != (val)) \ - (var) = (val); \ -} while (0) -#define virtqueue_clear_net_hdr(_hdr) do { \ - ASSIGN_UNLESS_EQUAL((_hdr)->csum_start, 0); \ - ASSIGN_UNLESS_EQUAL((_hdr)->csum_offset, 0); \ - ASSIGN_UNLESS_EQUAL((_hdr)->flags, 0); \ - ASSIGN_UNLESS_EQUAL((_hdr)->gso_type, 0); \ - ASSIGN_UNLESS_EQUAL((_hdr)->gso_size, 0); \ - ASSIGN_UNLESS_EQUAL((_hdr)->hdr_len, 0); \ -} while (0) - -static inline void -virtqueue_xmit_offload(struct virtio_net_hdr *hdr, - struct rte_mbuf *cookie, - bool offload) -{ - if (offload) { - if (cookie->ol_flags & PKT_TX_TCP_SEG) - cookie->ol_flags |= PKT_TX_TCP_CKSUM; - - switch (cookie->ol_flags & PKT_TX_L4_MASK) { - case PKT_TX_UDP_CKSUM: - hdr->csum_start = cookie->l2_len + cookie->l3_len; - hdr->csum_offset = offsetof(struct rte_udp_hdr, - dgram_cksum); - hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; - break; - - case PKT_TX_TCP_CKSUM: - hdr->csum_start = cookie->l2_len + cookie->l3_len; - hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum); - hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; - break; - - default: - ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); - ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); - ASSIGN_UNLESS_EQUAL(hdr->flags, 0); - break; - } - - /* TCP Segmentation Offload */ - if (cookie->ol_flags & PKT_TX_TCP_SEG) { - hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ? - VIRTIO_NET_HDR_GSO_TCPV6 : - VIRTIO_NET_HDR_GSO_TCPV4; - hdr->gso_size = cookie->tso_segsz; - hdr->hdr_len = - cookie->l2_len + - cookie->l3_len + - cookie->l4_len; - } else { - ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); - ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); - ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); - } - } -} static inline void virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, @@ -581,11 +444,11 @@ virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, uint16_t num) { struct vq_desc_extra *dxp; - struct virtqueue *vq = txvq->vq; + struct virtqueue *vq = virtnet_txq_to_vq(txvq); struct vring_desc *start_dp; struct virtio_net_hdr *hdr; uint16_t idx; - uint16_t head_size = vq->hw->vtnet_hdr_size; + int16_t head_size = vq->hw->vtnet_hdr_size; uint16_t i = 0; idx = vq->vq_desc_head_idx; @@ -596,21 +459,22 @@ virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; dxp->cookie = (void *)cookies[i]; dxp->ndescs = 1; + virtio_update_packet_stats(&txvq->stats, cookies[i]); - hdr = (struct virtio_net_hdr *) - rte_pktmbuf_prepend(cookies[i], head_size); - cookies[i]->pkt_len -= head_size; + hdr = rte_pktmbuf_mtod_offset(cookies[i], + struct virtio_net_hdr *, -head_size); /* if offload disabled, hdr is not zeroed yet, do it now */ if (!vq->hw->has_tx_offload) virtqueue_clear_net_hdr(hdr); else - virtqueue_xmit_offload(hdr, cookies[i], true); + virtqueue_xmit_offload(hdr, cookies[i]); - start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq); - start_dp[idx].len = cookies[i]->data_len; + start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size; + start_dp[idx].len = cookies[i]->data_len + head_size; start_dp[idx].flags = 0; + vq_update_avail_ring(vq, idx); idx++; @@ -626,11 +490,11 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, struct rte_mbuf *cookie, int in_order) { - struct virtqueue *vq = txvq->vq; + struct virtqueue *vq = virtnet_txq_to_vq(txvq); struct vring_packed_desc *dp; struct vq_desc_extra *dxp; uint16_t idx, id, flags; - uint16_t head_size = vq->hw->vtnet_hdr_size; + int16_t head_size = vq->hw->vtnet_hdr_size; struct virtio_net_hdr *hdr; id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; @@ -644,19 +508,18 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, flags = vq->vq_packed.cached_flags; /* prepend cannot fail, checked by caller */ - hdr = (struct virtio_net_hdr *) - rte_pktmbuf_prepend(cookie, head_size); - cookie->pkt_len -= head_size; + hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, + -head_size); /* if offload disabled, hdr is not zeroed yet, do it now */ if (!vq->hw->has_tx_offload) virtqueue_clear_net_hdr(hdr); else - virtqueue_xmit_offload(hdr, cookie, true); + virtqueue_xmit_offload(hdr, cookie); - dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); - dp->len = cookie->data_len; - dp->id = id; + dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size; + dp->len = cookie->data_len + head_size; + dp->id = id; if (++vq->vq_avail_idx >= vq->vq_nentries) { vq->vq_avail_idx -= vq->vq_nentries; @@ -671,100 +534,7 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; } - virtio_wmb(vq->hw->weak_barriers); - dp->flags = flags; -} - -static inline void -virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, - uint16_t needed, int can_push, int in_order) -{ - struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; - struct vq_desc_extra *dxp; - struct virtqueue *vq = txvq->vq; - struct vring_packed_desc *start_dp, *head_dp; - uint16_t idx, id, head_idx, head_flags; - uint16_t head_size = vq->hw->vtnet_hdr_size; - struct virtio_net_hdr *hdr; - uint16_t prev; - - id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; - - dxp = &vq->vq_descx[id]; - dxp->ndescs = needed; - dxp->cookie = cookie; - - head_idx = vq->vq_avail_idx; - idx = head_idx; - prev = head_idx; - start_dp = vq->vq_packed.ring.desc; - - head_dp = &vq->vq_packed.ring.desc[idx]; - head_flags = cookie->next ? VRING_DESC_F_NEXT : 0; - head_flags |= vq->vq_packed.cached_flags; - - if (can_push) { - /* prepend cannot fail, checked by caller */ - hdr = (struct virtio_net_hdr *) - rte_pktmbuf_prepend(cookie, head_size); - /* rte_pktmbuf_prepend() counts the hdr size to the pkt length, - * which is wrong. Below subtract restores correct pkt size. - */ - cookie->pkt_len -= head_size; - - /* if offload disabled, it is not zeroed below, do it now */ - if (!vq->hw->has_tx_offload) - virtqueue_clear_net_hdr(hdr); - } else { - /* setup first tx ring slot to point to header - * stored in reserved region. - */ - start_dp[idx].addr = txvq->virtio_net_hdr_mem + - RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); - start_dp[idx].len = vq->hw->vtnet_hdr_size; - hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; - idx++; - if (idx >= vq->vq_nentries) { - idx -= vq->vq_nentries; - vq->vq_packed.cached_flags ^= - VRING_PACKED_DESC_F_AVAIL_USED; - } - } - - virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload); - - do { - uint16_t flags; - - start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); - start_dp[idx].len = cookie->data_len; - if (likely(idx != head_idx)) { - flags = cookie->next ? VRING_DESC_F_NEXT : 0; - flags |= vq->vq_packed.cached_flags; - start_dp[idx].flags = flags; - } - prev = idx; - idx++; - if (idx >= vq->vq_nentries) { - idx -= vq->vq_nentries; - vq->vq_packed.cached_flags ^= - VRING_PACKED_DESC_F_AVAIL_USED; - } - } while ((cookie = cookie->next) != NULL); - - start_dp[prev].id = id; - - vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); - vq->vq_avail_idx = idx; - - if (!in_order) { - vq->vq_desc_head_idx = dxp->next; - if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; - } - - virtio_wmb(vq->hw->weak_barriers); - head_dp->flags = head_flags; + virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); } static inline void @@ -774,11 +544,12 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, { struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; struct vq_desc_extra *dxp; - struct virtqueue *vq = txvq->vq; + struct virtqueue *vq = virtnet_txq_to_vq(txvq); struct vring_desc *start_dp; uint16_t seg_num = cookie->nb_segs; uint16_t head_idx, idx; - uint16_t head_size = vq->hw->vtnet_hdr_size; + int16_t head_size = vq->hw->vtnet_hdr_size; + bool prepend_header = false; struct virtio_net_hdr *hdr; head_idx = vq->vq_desc_head_idx; @@ -794,12 +565,9 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, if (can_push) { /* prepend cannot fail, checked by caller */ - hdr = (struct virtio_net_hdr *) - rte_pktmbuf_prepend(cookie, head_size); - /* rte_pktmbuf_prepend() counts the hdr size to the pkt length, - * which is wrong. Below subtract restores correct pkt size. - */ - cookie->pkt_len -= head_size; + hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, + -head_size); + prepend_header = true; /* if offload disabled, it is not zeroed below, do it now */ if (!vq->hw->has_tx_offload) @@ -833,11 +601,17 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, idx = start_dp[idx].next; } - virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload); + if (vq->hw->has_tx_offload) + virtqueue_xmit_offload(hdr, cookie); do { - start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); - start_dp[idx].len = cookie->data_len; + start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); + start_dp[idx].len = cookie->data_len; + if (prepend_header) { + start_dp[idx].addr -= head_size; + start_dp[idx].len += head_size; + prepend_header = false; + } start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; idx = start_dp[idx].next; } while ((cookie = cookie->next) != NULL); @@ -861,9 +635,9 @@ virtio_dev_cq_start(struct rte_eth_dev *dev) { struct virtio_hw *hw = dev->data->dev_private; - if (hw->cvq && hw->cvq->vq) { + if (hw->cvq) { rte_spinlock_init(&hw->cvq->lock); - VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq->vq); + VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq)); } } @@ -872,18 +646,62 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id __rte_unused, - const struct rte_eth_rxconf *rx_conf __rte_unused, + const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { - uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; + uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; struct virtio_hw *hw = dev->data->dev_private; - struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtqueue *vq = hw->vqs[vq_idx]; struct virtnet_rx *rxvq; + uint16_t rx_free_thresh; + uint16_t buf_size; + const char *error; PMD_INIT_FUNC_TRACE(); - if (nb_desc == 0 || nb_desc > vq->vq_nentries) + if (rx_conf->rx_deferred_start) { + PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); + return -EINVAL; + } + + buf_size = virtio_rx_mem_pool_buf_size(mp); + if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size, + hw->rx_ol_scatter, &error)) { + PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s", + queue_idx, error); + return -EINVAL; + } + + rx_free_thresh = rx_conf->rx_free_thresh; + if (rx_free_thresh == 0) + rx_free_thresh = + RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH); + + if (rx_free_thresh & 0x3) { + PMD_INIT_LOG(ERR, "rx_free_thresh must be multiples of four." + " (rx_free_thresh=%u port=%u queue=%u)", + rx_free_thresh, dev->data->port_id, queue_idx); + return -EINVAL; + } + + if (rx_free_thresh >= vq->vq_nentries) { + PMD_INIT_LOG(ERR, "rx_free_thresh must be less than the " + "number of RX entries (%u)." + " (rx_free_thresh=%u port=%u queue=%u)", + vq->vq_nentries, + rx_free_thresh, dev->data->port_id, queue_idx); + return -EINVAL; + } + vq->vq_free_thresh = rx_free_thresh; + + /* + * For split ring vectorized path descriptors number must be + * equal to the ring size. + */ + if (nb_desc > vq->vq_nentries || + (!virtio_with_packed_queue(hw) && hw->use_vec_rx)) { nb_desc = vq->vq_nentries; + } vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); rxvq = &vq->rxq; @@ -897,20 +715,21 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, int virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) { - uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; + uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; struct virtio_hw *hw = dev->data->dev_private; - struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtqueue *vq = hw->vqs[vq_idx]; struct virtnet_rx *rxvq = &vq->rxq; struct rte_mbuf *m; uint16_t desc_idx; int error, nbufs, i; + bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); PMD_INIT_FUNC_TRACE(); /* Allocate blank mbufs for the each rx descriptor */ nbufs = 0; - if (hw->use_simple_rx) { + if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) { vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; @@ -921,19 +740,16 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) virtio_rxq_vec_setup(rxvq); } - memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf)); - for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; - desc_idx++) { - vq->sw_ring[vq->vq_nentries + desc_idx] = - &rxvq->fake_mbuf; - } + memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf)); + for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++) + vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf; - if (hw->use_simple_rx) { + if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { virtio_rxq_rearm_vec(rxvq); nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; } - } else if (hw->use_inorder_rx) { + } else if (!virtio_with_packed_queue(vq->hw) && in_order) { if ((!virtqueue_full(vq))) { uint16_t free_cnt = vq->vq_free_cnt; struct rte_mbuf *pkts[free_cnt]; @@ -946,10 +762,11 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) if (unlikely(error)) { for (i = 0; i < free_cnt; i++) rte_pktmbuf_free(pkts[i]); + } else { + nbufs += free_cnt; } } - nbufs += free_cnt; vq_update_avail_idx(vq); } } else { @@ -959,8 +776,8 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) break; /* Enqueue allocated buffers */ - if (vtpci_packed_queue(vq->hw)) - error = virtqueue_enqueue_recv_refill_packed(vq, + if (virtio_with_packed_queue(vq->hw)) + error = virtqueue_enqueue_recv_refill_packed_init(vq, &m, 1); else error = virtqueue_enqueue_recv_refill(vq, @@ -972,7 +789,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) nbufs++; } - if (!vtpci_packed_queue(vq->hw)) + if (!virtio_with_packed_queue(vq->hw)) vq_update_avail_idx(vq); } @@ -997,14 +814,19 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, unsigned int socket_id __rte_unused, const struct rte_eth_txconf *tx_conf) { - uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; + uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; struct virtio_hw *hw = dev->data->dev_private; - struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtqueue *vq = hw->vqs[vq_idx]; struct virtnet_tx *txvq; uint16_t tx_free_thresh; PMD_INIT_FUNC_TRACE(); + if (tx_conf->tx_deferred_start) { + PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); + return -EINVAL; + } + if (nb_desc == 0 || nb_desc > vq->vq_nentries) nb_desc = vq->vq_nentries; vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); @@ -1018,9 +840,9 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); if (tx_free_thresh >= (vq->vq_nentries - 3)) { - RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the " + PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the " "number of TX entries minus 3 (%u)." - " (tx_free_thresh=%u port=%u queue=%u)\n", + " (tx_free_thresh=%u port=%u queue=%u)", vq->vq_nentries - 3, tx_free_thresh, dev->data->port_id, queue_idx); return -EINVAL; @@ -1036,14 +858,14 @@ int virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) { - uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; + uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; struct virtio_hw *hw = dev->data->dev_private; - struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtqueue *vq = hw->vqs[vq_idx]; PMD_INIT_FUNC_TRACE(); - if (!vtpci_packed_queue(hw)) { - if (hw->use_inorder_tx) + if (!virtio_with_packed_queue(hw)) { + if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; } @@ -1060,13 +882,13 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) * Requeue the discarded mbuf. This should always be * successful since it was just dequeued. */ - if (vtpci_packed_queue(vq->hw)) + if (virtio_with_packed_queue(vq->hw)) error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); else error = virtqueue_enqueue_recv_refill(vq, &m, 1); if (unlikely(error)) { - RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf"); + PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); rte_pktmbuf_free(m); } } @@ -1078,53 +900,11 @@ virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) error = virtqueue_enqueue_refill_inorder(vq, &m, 1); if (unlikely(error)) { - RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf"); + PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); rte_pktmbuf_free(m); } } -static inline void -virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) -{ - uint32_t s = mbuf->pkt_len; - struct rte_ether_addr *ea; - - stats->bytes += s; - - if (s == 64) { - stats->size_bins[1]++; - } else if (s > 64 && s < 1024) { - uint32_t bin; - - /* count zeros, and offset into correct bin */ - bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; - stats->size_bins[bin]++; - } else { - if (s < 64) - stats->size_bins[0]++; - else if (s < 1519) - stats->size_bins[6]++; - else - stats->size_bins[7]++; - } - - ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); - if (rte_is_multicast_ether_addr(ea)) { - if (rte_is_broadcast_ether_addr(ea)) - stats->broadcast++; - else - stats->multicast++; - } -} - -static inline void -virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) -{ - VIRTIO_DUMP_PACKET(m, m->data_len); - - virtio_update_packet_stats(&rxvq->stats, m); -} - /* Optionally fill offload information in structure */ static inline int virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) @@ -1137,7 +917,7 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) return 0; - m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; + m->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN; ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); m->packet_type = ptype; @@ -1149,7 +929,7 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; if (hdr->csum_start <= hdrlen && l4_supported) { - m->ol_flags |= PKT_RX_L4_CKSUM_NONE; + m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_NONE; } else { /* Unknown proto or tunnel, do sw cksum. We can assume * the cksum field is in the first segment since the @@ -1159,9 +939,10 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) */ uint16_t csum = 0, off; - rte_raw_cksum_mbuf(m, hdr->csum_start, + if (rte_raw_cksum_mbuf(m, hdr->csum_start, rte_pktmbuf_pkt_len(m) - hdr->csum_start, - &csum); + &csum) < 0) + return -EINVAL; if (likely(csum != 0xffff)) csum = ~csum; off = hdr->csum_offset + hdr->csum_start; @@ -1170,7 +951,7 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) off) = csum; } } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { - m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; } /* GSO request, save required information in mbuf */ @@ -1181,13 +962,13 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) return -EINVAL; } - /* Update mss lengthes in mbuf */ + /* Update mss lengths in mbuf */ m->tso_segsz = hdr->gso_size; switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { case VIRTIO_NET_HDR_GSO_TCPV4: case VIRTIO_NET_HDR_GSO_TCPV6: - m->ol_flags |= PKT_RX_LRO | \ - PKT_RX_L4_CKSUM_NONE; + m->ol_flags |= RTE_MBUF_F_RX_LRO | + RTE_MBUF_F_RX_L4_CKSUM_NONE; break; default: return -EINVAL; @@ -1197,13 +978,12 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) return 0; } -#define VIRTIO_MBUF_BURST_SZ 64 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct virtnet_rx *rxvq = rx_queue; - struct virtqueue *vq = rxvq->vq; + struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm; uint16_t nb_used, num, nb_rx; @@ -1218,9 +998,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (unlikely(hw->started == 0)) return nb_rx; - nb_used = VIRTQUEUE_NUSED(vq); - - virtio_rmb(hw->weak_barriers); + nb_used = virtqueue_nused(vq); num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) @@ -1312,7 +1090,7 @@ virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct virtnet_rx *rxvq = rx_queue; - struct virtqueue *vq = rxvq->vq; + struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm; uint16_t num, nb_rx; @@ -1415,7 +1193,7 @@ virtio_recv_pkts_inorder(void *rx_queue, uint16_t nb_pkts) { struct virtnet_rx *rxvq = rx_queue; - struct virtqueue *vq = rxvq->vq; + struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm; struct rte_mbuf *prev = NULL; @@ -1433,12 +1211,10 @@ virtio_recv_pkts_inorder(void *rx_queue, if (unlikely(hw->started == 0)) return nb_rx; - nb_used = VIRTQUEUE_NUSED(vq); + nb_used = virtqueue_nused(vq); nb_used = RTE_MIN(nb_used, nb_pkts); nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); - virtio_rmb(hw->weak_barriers); - PMD_RX_LOG(DEBUG, "used:%d", nb_used); nb_enqueued = 0; @@ -1468,7 +1244,7 @@ virtio_recv_pkts_inorder(void *rx_queue, ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { seg_num = header->num_buffers; if (seg_num == 0) seg_num = 1; @@ -1527,8 +1303,7 @@ virtio_recv_pkts_inorder(void *rx_queue, uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, VIRTIO_MBUF_BURST_SZ); - if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { - virtio_rmb(hw->weak_barriers); + if (likely(virtqueue_nused(vq) >= rcv_cnt)) { num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, rcv_cnt); uint16_t extra_idx = 0; @@ -1602,7 +1377,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, uint16_t nb_pkts) { struct virtnet_rx *rxvq = rx_queue; - struct virtqueue *vq = rxvq->vq; + struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm; struct rte_mbuf *prev = NULL; @@ -1619,9 +1394,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, if (unlikely(hw->started == 0)) return nb_rx; - nb_used = VIRTQUEUE_NUSED(vq); - - virtio_rmb(hw->weak_barriers); + nb_used = virtqueue_nused(vq); PMD_RX_LOG(DEBUG, "used:%d", nb_used); @@ -1709,8 +1482,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, VIRTIO_MBUF_BURST_SZ); - if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { - virtio_rmb(hw->weak_barriers); + if (likely(virtqueue_nused(vq) >= rcv_cnt)) { num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, rcv_cnt); uint16_t extra_idx = 0; @@ -1783,7 +1555,7 @@ virtio_recv_mergeable_pkts_packed(void *rx_queue, uint16_t nb_pkts) { struct virtnet_rx *rxvq = rx_queue; - struct virtqueue *vq = rxvq->vq; + struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm; struct rte_mbuf *prev = NULL; @@ -1881,38 +1653,34 @@ virtio_recv_mergeable_pkts_packed(void *rx_queue, while (seg_res != 0) { uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, VIRTIO_MBUF_BURST_SZ); - if (likely(vq->vq_free_cnt >= rcv_cnt)) { - num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, - len, rcv_cnt); - uint16_t extra_idx = 0; + uint16_t extra_idx = 0; - rcv_cnt = num; - - while (extra_idx < rcv_cnt) { - rxm = rcv_pkts[extra_idx]; - - rxm->data_off = - RTE_PKTMBUF_HEADROOM - hdr_size; - rxm->pkt_len = (uint32_t)(len[extra_idx]); - rxm->data_len = (uint16_t)(len[extra_idx]); - - prev->next = rxm; - prev = rxm; - rx_pkts[nb_rx]->pkt_len += len[extra_idx]; - extra_idx += 1; - } - seg_res -= rcv_cnt; - if (!seg_res) { - virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); - nb_rx++; - } - } else { - PMD_RX_LOG(ERR, - "No enough segments for packet."); + rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, + len, rcv_cnt); + if (unlikely(rcv_cnt == 0)) { + PMD_RX_LOG(ERR, "No enough segments for packet."); rte_pktmbuf_free(rx_pkts[nb_rx]); rxvq->stats.errors++; break; } + + while (extra_idx < rcv_cnt) { + rxm = rcv_pkts[extra_idx]; + + rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; + rxm->pkt_len = (uint32_t)(len[extra_idx]); + rxm->data_len = (uint16_t)(len[extra_idx]); + + prev->next = rxm; + prev = rxm; + rx_pkts[nb_rx]->pkt_len += len[extra_idx]; + extra_idx += 1; + } + seg_res -= rcv_cnt; + if (!seg_res) { + virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); + nb_rx++; + } } rxvq->stats.packets += nb_rx; @@ -1967,7 +1735,7 @@ virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, #endif /* Do VLAN tag insertion */ - if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) { + if (unlikely(m->ol_flags & RTE_MBUF_F_TX_VLAN)) { error = rte_vlan_insert(&m); /* rte_vlan_insert() may change pointer * even in the case of failure @@ -1986,7 +1754,7 @@ virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, break; } - if (m->ol_flags & PKT_TX_TCP_SEG) + if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) virtio_tso_fix_cksum(m); } @@ -1998,11 +1766,11 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct virtnet_tx *txvq = tx_queue; - struct virtqueue *vq = txvq->vq; + struct virtqueue *vq = virtnet_txq_to_vq(txvq); struct virtio_hw *hw = vq->hw; uint16_t hdr_size = hw->vtnet_hdr_size; uint16_t nb_tx = 0; - bool in_order = hw->use_inorder_tx; + bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) return nb_tx; @@ -2018,11 +1786,11 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { struct rte_mbuf *txm = tx_pkts[nb_tx]; - int can_push = 0, slots, need; + int can_push = 0, use_indirect = 0, slots, need; /* optimize ring usage */ - if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || - vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && + if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || + virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && rte_mbuf_refcnt_read(txm) == 1 && RTE_MBUF_DIRECT(txm) && txm->nb_segs == 1 && @@ -2030,12 +1798,15 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, rte_is_aligned(rte_pktmbuf_mtod(txm, char *), __alignof__(struct virtio_net_hdr_mrg_rxbuf))) can_push = 1; - + else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && + txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) + use_indirect = 1; /* How many main ring entries are needed to this Tx? + * indirect => 1 * any_layout => number of segments * default => number of segments + 1 */ - slots = txm->nb_segs + !can_push; + slots = use_indirect ? 1 : (txm->nb_segs + !can_push); need = slots - vq->vq_free_cnt; /* Positive value indicates it need free vring descriptors */ @@ -2053,7 +1824,8 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, if (can_push) virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); else - virtqueue_enqueue_xmit_packed(txvq, txm, slots, 0, + virtqueue_enqueue_xmit_packed(txvq, txm, slots, + use_indirect, 0, in_order); virtio_update_packet_stats(&txvq->stats, txm); @@ -2075,7 +1847,7 @@ uint16_t virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct virtnet_tx *txvq = tx_queue; - struct virtqueue *vq = txvq->vq; + struct virtqueue *vq = virtnet_txq_to_vq(txvq); struct virtio_hw *hw = vq->hw; uint16_t hdr_size = hw->vtnet_hdr_size; uint16_t nb_used, nb_tx = 0; @@ -2087,9 +1859,9 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) return nb_pkts; PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); - nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(hw->weak_barriers); + nb_used = virtqueue_nused(vq); + if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) virtio_xmit_cleanup(vq, nb_used); @@ -2098,8 +1870,8 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) int can_push = 0, use_indirect = 0, slots, need; /* optimize ring usage */ - if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || - vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && + if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || + virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && rte_mbuf_refcnt_read(txm) == 1 && RTE_MBUF_DIRECT(txm) && txm->nb_segs == 1 && @@ -2107,7 +1879,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) rte_is_aligned(rte_pktmbuf_mtod(txm, char *), __alignof__(struct virtio_net_hdr_mrg_rxbuf))) can_push = 1; - else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && + else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) use_indirect = 1; @@ -2121,8 +1893,8 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* Positive value indicates it need free vring descriptors */ if (unlikely(need > 0)) { - nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(hw->weak_barriers); + nb_used = virtqueue_nused(vq); + need = RTE_MIN(need, (int)nb_used); virtio_xmit_cleanup(vq, need); @@ -2155,17 +1927,32 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) return nb_tx; } +static __rte_always_inline int +virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) +{ + uint16_t nb_used, nb_clean, nb_descs; + + nb_descs = vq->vq_free_cnt + need; + nb_used = virtqueue_nused(vq); + nb_clean = RTE_MIN(need, (int)nb_used); + + virtio_xmit_cleanup_inorder(vq, nb_clean); + + return nb_descs - vq->vq_free_cnt; +} + uint16_t virtio_xmit_pkts_inorder(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct virtnet_tx *txvq = tx_queue; - struct virtqueue *vq = txvq->vq; + struct virtqueue *vq = virtnet_txq_to_vq(txvq); struct virtio_hw *hw = vq->hw; uint16_t hdr_size = hw->vtnet_hdr_size; - uint16_t nb_used, nb_avail, nb_tx = 0, nb_inorder_pkts = 0; + uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; struct rte_mbuf *inorder_pkts[nb_pkts]; + int need; if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) return nb_tx; @@ -2175,24 +1962,18 @@ virtio_xmit_pkts_inorder(void *tx_queue, VIRTQUEUE_DUMP(vq); PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); - nb_used = VIRTQUEUE_NUSED(vq); + nb_used = virtqueue_nused(vq); - virtio_rmb(hw->weak_barriers); if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) virtio_xmit_cleanup_inorder(vq, nb_used); - if (unlikely(!vq->vq_free_cnt)) - virtio_xmit_cleanup_inorder(vq, nb_used); - - nb_avail = RTE_MIN(vq->vq_free_cnt, nb_pkts); - - for (nb_tx = 0; nb_tx < nb_avail; nb_tx++) { + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { struct rte_mbuf *txm = tx_pkts[nb_tx]; - int slots, need; + int slots; /* optimize ring usage */ - if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || - vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && + if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || + virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && rte_mbuf_refcnt_read(txm) == 1 && RTE_MBUF_DIRECT(txm) && txm->nb_segs == 1 && @@ -2202,11 +1983,21 @@ virtio_xmit_pkts_inorder(void *tx_queue, inorder_pkts[nb_inorder_pkts] = txm; nb_inorder_pkts++; - virtio_update_packet_stats(&txvq->stats, txm); continue; } if (nb_inorder_pkts) { + need = nb_inorder_pkts - vq->vq_free_cnt; + if (unlikely(need > 0)) { + need = virtio_xmit_try_cleanup_inorder(vq, + need); + if (unlikely(need > 0)) { + PMD_TX_LOG(ERR, + "No free tx descriptors to " + "transmit"); + break; + } + } virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, nb_inorder_pkts); nb_inorder_pkts = 0; @@ -2215,13 +2006,7 @@ virtio_xmit_pkts_inorder(void *tx_queue, slots = txm->nb_segs + 1; need = slots - vq->vq_free_cnt; if (unlikely(need > 0)) { - nb_used = VIRTQUEUE_NUSED(vq); - virtio_rmb(hw->weak_barriers); - need = RTE_MIN(need, (int)nb_used); - - virtio_xmit_cleanup_inorder(vq, need); - - need = slots - vq->vq_free_cnt; + need = virtio_xmit_try_cleanup_inorder(vq, slots); if (unlikely(need > 0)) { PMD_TX_LOG(ERR, @@ -2236,9 +2021,22 @@ virtio_xmit_pkts_inorder(void *tx_queue, } /* Transmit all inorder packets */ - if (nb_inorder_pkts) + if (nb_inorder_pkts) { + need = nb_inorder_pkts - vq->vq_free_cnt; + if (unlikely(need > 0)) { + need = virtio_xmit_try_cleanup_inorder(vq, + need); + if (unlikely(need > 0)) { + PMD_TX_LOG(ERR, + "No free tx descriptors to transmit"); + nb_inorder_pkts = vq->vq_free_cnt; + nb_tx -= need; + } + } + virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, nb_inorder_pkts); + } txvq->stats.packets += nb_tx; @@ -2255,3 +2053,19 @@ virtio_xmit_pkts_inorder(void *tx_queue, return nb_tx; } + +__rte_weak uint16_t +virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, + struct rte_mbuf **rx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + return 0; +} + +__rte_weak uint16_t +virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, + struct rte_mbuf **tx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + return 0; +}