X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtio_rxtx.c;h=854af399e8062f72d7f15c9e083fae5f452b9b5d;hb=f234e5bd996dea4ad86e0f7ad48e749e72bdbf42;hp=f326222dedc5fc854cc360f4efd2d3cc0b7b5bf7;hpb=2c0eb46f51b3d1364379243b5c9accb3ba49725d;p=dpdk.git diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index f326222ded..854af399e8 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation */ #include @@ -39,17 +10,20 @@ #include #include -#include #include #include #include #include #include -#include +#include #include #include #include #include +#include +#include +#include +#include #include "virtio_logs.h" #include "virtio_ethdev.h" @@ -67,11 +41,16 @@ #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \ ETH_TXQ_FLAGS_NOOFFLOADS) -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 -static int use_simple_rxtx; -#endif +int +virtio_dev_rx_queue_done(void *rxq, uint16_t offset) +{ + struct virtnet_rx *rxvq = rxq; + struct virtqueue *vq = rxvq->vq; -static void + return VIRTQUEUE_NUSED(vq) >= offset; +} + +void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) { struct vring_desc *dp, *dp_tail; @@ -123,7 +102,7 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; if (unlikely(cookie == NULL)) { - PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n", + PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", vq->vq_used_cons_idx); break; } @@ -193,8 +172,8 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) start_dp = vq->vq_ring.desc; start_dp[idx].addr = - (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM - - hw->vtnet_hdr_size); + VIRTIO_MBUF_ADDR(cookie, vq) + + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; start_dp[idx].len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; start_dp[idx].flags = VRING_DESC_F_WRITE; @@ -208,30 +187,101 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) return 0; } +/* When doing TSO, the IP length is not included in the pseudo header + * checksum of the packet given to the PMD, but for virtio it is + * expected. + */ +static void +virtio_tso_fix_cksum(struct rte_mbuf *m) +{ + /* common case: header is not fragmented */ + if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + + m->l4_len)) { + struct ipv4_hdr *iph; + struct ipv6_hdr *ip6h; + struct tcp_hdr *th; + uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; + uint32_t tmp; + + iph = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len); + th = RTE_PTR_ADD(iph, m->l3_len); + if ((iph->version_ihl >> 4) == 4) { + iph->hdr_checksum = 0; + iph->hdr_checksum = rte_ipv4_cksum(iph); + ip_len = iph->total_length; + ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - + m->l3_len); + } else { + ip6h = (struct ipv6_hdr *)iph; + ip_paylen = ip6h->payload_len; + } + + /* calculate the new phdr checksum not including ip_paylen */ + prev_cksum = th->cksum; + tmp = prev_cksum; + tmp += ip_paylen; + tmp = (tmp & 0xffff) + (tmp >> 16); + new_cksum = tmp; + + /* replace it in the packet */ + th->cksum = new_cksum; + } +} + +static inline int +tx_offload_enabled(struct virtio_hw *hw) +{ + return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) || + vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) || + vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6); +} + +/* avoid write operation when necessary, to lessen cache issues */ +#define ASSIGN_UNLESS_EQUAL(var, val) do { \ + if ((var) != (val)) \ + (var) = (val); \ +} while (0) + static inline void -virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie, +virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, uint16_t needed, int use_indirect, int can_push) { + struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; struct vq_desc_extra *dxp; + struct virtqueue *vq = txvq->vq; struct vring_desc *start_dp; uint16_t seg_num = cookie->nb_segs; uint16_t head_idx, idx; - uint16_t head_size = txvq->hw->vtnet_hdr_size; - unsigned long offs; + uint16_t head_size = vq->hw->vtnet_hdr_size; + struct virtio_net_hdr *hdr; + int offload; - head_idx = txvq->vq_desc_head_idx; + offload = tx_offload_enabled(vq->hw); + head_idx = vq->vq_desc_head_idx; idx = head_idx; - dxp = &txvq->vq_descx[idx]; + dxp = &vq->vq_descx[idx]; dxp->cookie = (void *)cookie; dxp->ndescs = needed; - start_dp = txvq->vq_ring.desc; + start_dp = vq->vq_ring.desc; if (can_push) { - /* put on zero'd transmit header (no offloads) */ - void *hdr = rte_pktmbuf_prepend(cookie, head_size); - - memset(hdr, 0, head_size); + /* prepend cannot fail, checked by caller */ + hdr = (struct virtio_net_hdr *) + rte_pktmbuf_prepend(cookie, head_size); + /* rte_pktmbuf_prepend() counts the hdr size to the pkt length, + * which is wrong. Below subtract restores correct pkt size. + */ + cookie->pkt_len -= head_size; + /* if offload disabled, it is not zeroed below, do it now */ + if (offload == 0) { + ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); + ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); + ASSIGN_UNLESS_EQUAL(hdr->flags, 0); + ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); + ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); + ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); + } } else if (use_indirect) { /* setup tx ring slot to point to indirect * descriptor list stored in reserved region. @@ -239,14 +289,11 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie, * the first slot in indirect ring is already preset * to point to the header in reserved region */ - struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; - - offs = idx * sizeof(struct virtio_tx_region) - + offsetof(struct virtio_tx_region, tx_indir); - - start_dp[idx].addr = txvq->virtio_net_hdr_mem + offs; + start_dp[idx].addr = txvq->virtio_net_hdr_mem + + RTE_PTR_DIFF(&txr[idx].tx_indir, txr); start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); start_dp[idx].flags = VRING_DESC_F_INDIRECT; + hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; /* loop below will fill in rest of the indirect elements */ start_dp = txr[idx].tx_indir; @@ -255,214 +302,177 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie, /* setup first tx ring slot to point to header * stored in reserved region. */ - offs = idx * sizeof(struct virtio_tx_region) - + offsetof(struct virtio_tx_region, tx_hdr); - - start_dp[idx].addr = txvq->virtio_net_hdr_mem + offs; - start_dp[idx].len = txvq->hw->vtnet_hdr_size; + start_dp[idx].addr = txvq->virtio_net_hdr_mem + + RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); + start_dp[idx].len = vq->hw->vtnet_hdr_size; start_dp[idx].flags = VRING_DESC_F_NEXT; + hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; + idx = start_dp[idx].next; } + /* Checksum Offload / TSO */ + if (offload) { + if (cookie->ol_flags & PKT_TX_TCP_SEG) + cookie->ol_flags |= PKT_TX_TCP_CKSUM; + + switch (cookie->ol_flags & PKT_TX_L4_MASK) { + case PKT_TX_UDP_CKSUM: + hdr->csum_start = cookie->l2_len + cookie->l3_len; + hdr->csum_offset = offsetof(struct udp_hdr, + dgram_cksum); + hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; + break; + + case PKT_TX_TCP_CKSUM: + hdr->csum_start = cookie->l2_len + cookie->l3_len; + hdr->csum_offset = offsetof(struct tcp_hdr, cksum); + hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; + break; + + default: + ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); + ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); + ASSIGN_UNLESS_EQUAL(hdr->flags, 0); + break; + } + + /* TCP Segmentation Offload */ + if (cookie->ol_flags & PKT_TX_TCP_SEG) { + virtio_tso_fix_cksum(cookie); + hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ? + VIRTIO_NET_HDR_GSO_TCPV6 : + VIRTIO_NET_HDR_GSO_TCPV4; + hdr->gso_size = cookie->tso_segsz; + hdr->hdr_len = + cookie->l2_len + + cookie->l3_len + + cookie->l4_len; + } else { + ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); + ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); + ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); + } + } + do { - start_dp[idx].addr = rte_mbuf_data_dma_addr(cookie); + start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); start_dp[idx].len = cookie->data_len; start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; idx = start_dp[idx].next; } while ((cookie = cookie->next) != NULL); if (use_indirect) - idx = txvq->vq_ring.desc[head_idx].next; + idx = vq->vq_ring.desc[head_idx].next; - txvq->vq_desc_head_idx = idx; - if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) - txvq->vq_desc_tail_idx = idx; - txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed); - vq_update_avail_ring(txvq, head_idx); + vq->vq_desc_head_idx = idx; + if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) + vq->vq_desc_tail_idx = idx; + vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); + vq_update_avail_ring(vq, head_idx); } -static inline struct rte_mbuf * -rte_rxmbuf_alloc(struct rte_mempool *mp) +void +virtio_dev_cq_start(struct rte_eth_dev *dev) { - struct rte_mbuf *m; - - m = __rte_mbuf_raw_alloc(mp); - __rte_mbuf_sanity_check_raw(m, 0); + struct virtio_hw *hw = dev->data->dev_private; - return m; + if (hw->cvq && hw->cvq->vq) { + rte_spinlock_init(&hw->cvq->lock); + VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq->vq); + } } -static void -virtio_dev_vring_start(struct virtqueue *vq, int queue_type) +int +virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id __rte_unused, + __rte_unused const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp) { - struct rte_mbuf *m; - int i, nbufs, error, size = vq->vq_nentries; - struct vring *vr = &vq->vq_ring; - uint8_t *ring_mem = vq->vq_ring_virt_mem; + uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; + struct virtio_hw *hw = dev->data->dev_private; + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtnet_rx *rxvq; PMD_INIT_FUNC_TRACE(); - /* - * Reinitialise since virtio port might have been stopped and restarted - */ - memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size); - vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN); - vq->vq_used_cons_idx = 0; - vq->vq_desc_head_idx = 0; - vq->vq_avail_idx = 0; - vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); - vq->vq_free_cnt = vq->vq_nentries; - memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); - - vring_desc_init(vr->desc, size); - - /* - * Disable device(host) interrupting guest - */ - virtqueue_disable_intr(vq); - - /* Only rx virtqueue needs mbufs to be allocated at initialization */ - if (queue_type == VTNET_RQ) { - if (vq->mpool == NULL) - rte_exit(EXIT_FAILURE, - "Cannot allocate initial mbufs for rx virtqueue"); - - /* Allocate blank mbufs for the each rx descriptor */ - nbufs = 0; - error = ENOSPC; - -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - if (use_simple_rxtx) - for (i = 0; i < vq->vq_nentries; i++) { - vq->vq_ring.avail->ring[i] = i; - vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE; - } -#endif - memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf)); - for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++) - vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf; - - while (!virtqueue_full(vq)) { - m = rte_rxmbuf_alloc(vq->mpool); - if (m == NULL) - break; - - /****************************************** - * Enqueue allocated buffers * - *******************************************/ -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - if (use_simple_rxtx) - error = virtqueue_enqueue_recv_refill_simple(vq, m); - else -#endif - error = virtqueue_enqueue_recv_refill(vq, m); - if (error) { - rte_pktmbuf_free(m); - break; - } - nbufs++; - } - - vq_update_avail_idx(vq); + if (nb_desc == 0 || nb_desc > vq->vq_nentries) + nb_desc = vq->vq_nentries; + vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); - PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); - } else if (queue_type == VTNET_TQ) { -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - if (use_simple_rxtx) { - int mid_idx = vq->vq_nentries >> 1; - for (i = 0; i < mid_idx; i++) { - vq->vq_ring.avail->ring[i] = i + mid_idx; - vq->vq_ring.desc[i + mid_idx].next = i; - vq->vq_ring.desc[i + mid_idx].addr = - vq->virtio_net_hdr_mem + - offsetof(struct virtio_tx_region, tx_hdr); - vq->vq_ring.desc[i + mid_idx].len = - vq->hw->vtnet_hdr_size; - vq->vq_ring.desc[i + mid_idx].flags = - VRING_DESC_F_NEXT; - vq->vq_ring.desc[i].flags = 0; - } - for (i = mid_idx; i < vq->vq_nentries; i++) - vq->vq_ring.avail->ring[i] = i; - } -#endif + rxvq = &vq->rxq; + rxvq->queue_id = queue_idx; + rxvq->mpool = mp; + if (rxvq->mpool == NULL) { + rte_exit(EXIT_FAILURE, + "Cannot allocate mbufs for rx virtqueue"); } -} + dev->data->rx_queues[queue_idx] = rxvq; -void -virtio_dev_cq_start(struct rte_eth_dev *dev) -{ - struct virtio_hw *hw = dev->data->dev_private; - - if (hw->cvq) { - virtio_dev_vring_start(hw->cvq, VTNET_CQ); - VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq); - } + return 0; } -void -virtio_dev_rxtx_start(struct rte_eth_dev *dev) +int +virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) { - /* - * Start receive and transmit vrings - * - Setup vring structure for all queues - * - Initialize descriptor for the rx vring - * - Allocate blank mbufs for the each rx descriptor - * - */ - int i; + uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; + struct virtio_hw *hw = dev->data->dev_private; + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtnet_rx *rxvq = &vq->rxq; + struct rte_mbuf *m; + uint16_t desc_idx; + int error, nbufs; PMD_INIT_FUNC_TRACE(); - /* Start rx vring. */ - for (i = 0; i < dev->data->nb_rx_queues; i++) { - virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); + /* Allocate blank mbufs for the each rx descriptor */ + nbufs = 0; + + if (hw->use_simple_rx) { + for (desc_idx = 0; desc_idx < vq->vq_nentries; + desc_idx++) { + vq->vq_ring.avail->ring[desc_idx] = desc_idx; + vq->vq_ring.desc[desc_idx].flags = + VRING_DESC_F_WRITE; + } } - /* Start tx vring. */ - for (i = 0; i < dev->data->nb_tx_queues; i++) { - virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); + memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf)); + for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; + desc_idx++) { + vq->sw_ring[vq->vq_nentries + desc_idx] = + &rxvq->fake_mbuf; } -} -int -virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, - uint16_t queue_idx, - uint16_t nb_desc, - unsigned int socket_id, - __rte_unused const struct rte_eth_rxconf *rx_conf, - struct rte_mempool *mp) -{ - uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; - struct virtqueue *vq; - int ret; + while (!virtqueue_full(vq)) { + m = rte_mbuf_raw_alloc(rxvq->mpool); + if (m == NULL) + break; - PMD_INIT_FUNC_TRACE(); - ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx, - nb_desc, socket_id, &vq); - if (ret < 0) { - PMD_INIT_LOG(ERR, "rvq initialization failed"); - return ret; + /* Enqueue allocated buffers */ + if (hw->use_simple_rx) + error = virtqueue_enqueue_recv_refill_simple(vq, m); + else + error = virtqueue_enqueue_recv_refill(vq, m); + + if (error) { + rte_pktmbuf_free(m); + break; + } + nbufs++; } - /* Create mempool for rx mbuf allocation */ - vq->mpool = mp; + vq_update_avail_idx(vq); - dev->data->rx_queues[queue_idx] = vq; + PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - virtio_rxq_vec_setup(vq); -#endif + virtio_rxq_vec_setup(rxvq); - return 0; -} + VIRTQUEUE_DUMP(vq); -void -virtio_dev_rx_queue_release(void *rxq) -{ - virtio_dev_queue_release(rxq); + return 0; } /* @@ -476,43 +486,27 @@ int virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, - unsigned int socket_id, + unsigned int socket_id __rte_unused, const struct rte_eth_txconf *tx_conf) { uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; - -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 struct virtio_hw *hw = dev->data->dev_private; -#endif - struct virtqueue *vq; + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtnet_tx *txvq; uint16_t tx_free_thresh; - int ret; PMD_INIT_FUNC_TRACE(); - if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS) - != ETH_TXQ_FLAGS_NOXSUMS) { - PMD_INIT_LOG(ERR, "TX checksum offload not supported\n"); - return -EINVAL; - } + /* cannot use simple rxtx funcs with multisegs or offloads */ + if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) != VIRTIO_SIMPLE_FLAGS) + hw->use_simple_tx = 0; -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - /* Use simple rx/tx func if single segment and no offloads */ - if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS && - !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { - PMD_INIT_LOG(INFO, "Using simple rx/tx path"); - dev->tx_pkt_burst = virtio_xmit_pkts_simple; - dev->rx_pkt_burst = virtio_recv_pkts_vec; - use_simple_rxtx = 1; - } -#endif + if (nb_desc == 0 || nb_desc > vq->vq_nentries) + nb_desc = vq->vq_nentries; + vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); - ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx, - nb_desc, socket_id, &vq); - if (ret < 0) { - PMD_INIT_LOG(ERR, "rvq initialization failed"); - return ret; - } + txvq = &vq->txq; + txvq->queue_id = queue_idx; tx_free_thresh = tx_conf->tx_free_thresh; if (tx_free_thresh == 0) @@ -530,14 +524,46 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, vq->vq_free_thresh = tx_free_thresh; - dev->data->tx_queues[queue_idx] = vq; + dev->data->tx_queues[queue_idx] = txvq; return 0; } -void -virtio_dev_tx_queue_release(void *txq) +int +virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, + uint16_t queue_idx) { - virtio_dev_queue_release(txq); + uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; + struct virtio_hw *hw = dev->data->dev_private; + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + uint16_t mid_idx = vq->vq_nentries >> 1; + struct virtnet_tx *txvq = &vq->txq; + uint16_t desc_idx; + + PMD_INIT_FUNC_TRACE(); + + if (hw->use_simple_tx) { + for (desc_idx = 0; desc_idx < mid_idx; desc_idx++) { + vq->vq_ring.avail->ring[desc_idx] = + desc_idx + mid_idx; + vq->vq_ring.desc[desc_idx + mid_idx].next = + desc_idx; + vq->vq_ring.desc[desc_idx + mid_idx].addr = + txvq->virtio_net_hdr_mem + + offsetof(struct virtio_tx_region, tx_hdr); + vq->vq_ring.desc[desc_idx + mid_idx].len = + vq->hw->vtnet_hdr_size; + vq->vq_ring.desc[desc_idx + mid_idx].flags = + VRING_DESC_F_NEXT; + vq->vq_ring.desc[desc_idx].flags = 0; + } + for (desc_idx = mid_idx; desc_idx < vq->vq_nentries; + desc_idx++) + vq->vq_ring.avail->ring[desc_idx] = desc_idx; + } + + VIRTQUEUE_DUMP(vq); + + return 0; } static void @@ -556,35 +582,115 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) } static void -virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf) +virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) { uint32_t s = mbuf->pkt_len; struct ether_addr *ea; if (s == 64) { - vq->size_bins[1]++; + stats->size_bins[1]++; } else if (s > 64 && s < 1024) { uint32_t bin; /* count zeros, and offset into correct bin */ bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; - vq->size_bins[bin]++; + stats->size_bins[bin]++; } else { if (s < 64) - vq->size_bins[0]++; + stats->size_bins[0]++; else if (s < 1519) - vq->size_bins[6]++; + stats->size_bins[6]++; else if (s >= 1519) - vq->size_bins[7]++; + stats->size_bins[7]++; } ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *); if (is_multicast_ether_addr(ea)) { if (is_broadcast_ether_addr(ea)) - vq->broadcast++; + stats->broadcast++; else - vq->multicast++; + stats->multicast++; + } +} + +/* Optionally fill offload information in structure */ +static int +virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) +{ + struct rte_net_hdr_lens hdr_lens; + uint32_t hdrlen, ptype; + int l4_supported = 0; + + /* nothing to do */ + if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) + return 0; + + m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; + + ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); + m->packet_type = ptype; + if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || + (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || + (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) + l4_supported = 1; + + if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { + hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; + if (hdr->csum_start <= hdrlen && l4_supported) { + m->ol_flags |= PKT_RX_L4_CKSUM_NONE; + } else { + /* Unknown proto or tunnel, do sw cksum. We can assume + * the cksum field is in the first segment since the + * buffers we provided to the host are large enough. + * In case of SCTP, this will be wrong since it's a CRC + * but there's nothing we can do. + */ + uint16_t csum = 0, off; + + rte_raw_cksum_mbuf(m, hdr->csum_start, + rte_pktmbuf_pkt_len(m) - hdr->csum_start, + &csum); + if (likely(csum != 0xffff)) + csum = ~csum; + off = hdr->csum_offset + hdr->csum_start; + if (rte_pktmbuf_data_len(m) >= off + 1) + *rte_pktmbuf_mtod_offset(m, uint16_t *, + off) = csum; + } + } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { + m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; } + + /* GSO request, save required information in mbuf */ + if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { + /* Check unsupported modes */ + if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || + (hdr->gso_size == 0)) { + return -EINVAL; + } + + /* Update mss lengthes in mbuf */ + m->tso_segsz = hdr->gso_size; + switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { + case VIRTIO_NET_HDR_GSO_TCPV4: + case VIRTIO_NET_HDR_GSO_TCPV6: + m->ol_flags |= PKT_RX_LRO | \ + PKT_RX_L4_CKSUM_NONE; + break; + default: + return -EINVAL; + } + } + + return 0; +} + +static inline int +rx_offload_enabled(struct virtio_hw *hw) +{ + return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) || + vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || + vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6); } #define VIRTIO_MBUF_BURST_SZ 64 @@ -592,8 +698,9 @@ virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf) uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { - struct virtqueue *rxvq = rx_queue; - struct virtio_hw *hw; + struct virtnet_rx *rxvq = rx_queue; + struct virtqueue *vq = rxvq->vq; + struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm, *new_mbuf; uint16_t nb_used, num, nb_rx; uint32_t len[VIRTIO_MBUF_BURST_SZ]; @@ -601,23 +708,29 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) int error; uint32_t i, nb_enqueued; uint32_t hdr_size; + int offload; + struct virtio_net_hdr *hdr; + + nb_rx = 0; + if (unlikely(hw->started == 0)) + return nb_rx; - nb_used = VIRTQUEUE_NUSED(rxvq); + nb_used = VIRTQUEUE_NUSED(vq); virtio_rmb(); - num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts); - num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ); + num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; + if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) + num = VIRTIO_MBUF_BURST_SZ; if (likely(num > DESC_PER_CACHELINE)) - num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); + num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); - num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num); + num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); - hw = rxvq->hw; - nb_rx = 0; nb_enqueued = 0; hdr_size = hw->vtnet_hdr_size; + offload = rx_offload_enabled(hw); for (i = 0; i < num ; i++) { rxm = rcv_pkts[i]; @@ -627,8 +740,8 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) { PMD_RX_LOG(ERR, "Packet drop"); nb_enqueued++; - virtio_discard_rxbuf(rxvq, rxm); - rxvq->errors++; + virtio_discard_rxbuf(vq, rxm); + rxvq->stats.errors++; continue; } @@ -637,35 +750,42 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->ol_flags = 0; rxm->vlan_tci = 0; - rxm->nb_segs = 1; - rxm->next = NULL; rxm->pkt_len = (uint32_t)(len[i] - hdr_size); rxm->data_len = (uint16_t)(len[i] - hdr_size); + hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + + RTE_PKTMBUF_HEADROOM - hdr_size); + if (hw->vlan_strip) rte_vlan_strip(rxm); + if (offload && virtio_rx_offload(rxm, hdr) < 0) { + virtio_discard_rxbuf(vq, rxm); + rxvq->stats.errors++; + continue; + } + VIRTIO_DUMP_PACKET(rxm, rxm->data_len); rx_pkts[nb_rx++] = rxm; - rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len; - virtio_update_packet_stats(rxvq, rxm); + rxvq->stats.bytes += rxm->pkt_len; + virtio_update_packet_stats(&rxvq->stats, rxm); } - rxvq->packets += nb_rx; + rxvq->stats.packets += nb_rx; /* Allocate new mbuf for the used descriptor */ error = ENOSPC; - while (likely(!virtqueue_full(rxvq))) { - new_mbuf = rte_rxmbuf_alloc(rxvq->mpool); + while (likely(!virtqueue_full(vq))) { + new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool); if (unlikely(new_mbuf == NULL)) { struct rte_eth_dev *dev = &rte_eth_devices[rxvq->port_id]; dev->data->rx_mbuf_alloc_failed++; break; } - error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf); + error = virtqueue_enqueue_recv_refill(vq, new_mbuf); if (unlikely(error)) { rte_pktmbuf_free(new_mbuf); break; @@ -674,10 +794,10 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) } if (likely(nb_enqueued)) { - vq_update_avail_idx(rxvq); + vq_update_avail_idx(vq); - if (unlikely(virtqueue_kick_prepare(rxvq))) { - virtqueue_notify(rxvq); + if (unlikely(virtqueue_kick_prepare(vq))) { + virtqueue_notify(vq); PMD_RX_LOG(DEBUG, "Notified"); } } @@ -690,8 +810,9 @@ virtio_recv_mergeable_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { - struct virtqueue *rxvq = rx_queue; - struct virtio_hw *hw; + struct virtnet_rx *rxvq = rx_queue; + struct virtqueue *vq = rxvq->vq; + struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm, *new_mbuf; uint16_t nb_used, num, nb_rx; uint32_t len[VIRTIO_MBUF_BURST_SZ]; @@ -703,21 +824,25 @@ virtio_recv_mergeable_pkts(void *rx_queue, uint16_t extra_idx; uint32_t seg_res; uint32_t hdr_size; + int offload; + + nb_rx = 0; + if (unlikely(hw->started == 0)) + return nb_rx; - nb_used = VIRTQUEUE_NUSED(rxvq); + nb_used = VIRTQUEUE_NUSED(vq); virtio_rmb(); PMD_RX_LOG(DEBUG, "used:%d", nb_used); - hw = rxvq->hw; - nb_rx = 0; i = 0; nb_enqueued = 0; seg_num = 0; extra_idx = 0; seg_res = 0; hdr_size = hw->vtnet_hdr_size; + offload = rx_offload_enabled(hw); while (i < nb_used) { struct virtio_net_hdr_mrg_rxbuf *header; @@ -725,7 +850,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, if (nb_rx == nb_pkts) break; - num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1); + num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1); if (num != 1) continue; @@ -739,8 +864,8 @@ virtio_recv_mergeable_pkts(void *rx_queue, if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) { PMD_RX_LOG(ERR, "Packet drop"); nb_enqueued++; - virtio_discard_rxbuf(rxvq, rxm); - rxvq->errors++; + virtio_discard_rxbuf(vq, rxm); + rxvq->stats.errors++; continue; } @@ -753,7 +878,6 @@ virtio_recv_mergeable_pkts(void *rx_queue, rxm->data_off = RTE_PKTMBUF_HEADROOM; rxm->nb_segs = seg_num; - rxm->next = NULL; rxm->ol_flags = 0; rxm->vlan_tci = 0; rxm->pkt_len = (uint32_t)(len[0] - hdr_size); @@ -763,6 +887,12 @@ virtio_recv_mergeable_pkts(void *rx_queue, rx_pkts[nb_rx] = rxm; prev = rxm; + if (offload && virtio_rx_offload(rxm, &header->hdr) < 0) { + virtio_discard_rxbuf(vq, rxm); + rxvq->stats.errors++; + continue; + } + seg_res = seg_num - 1; while (seg_res != 0) { @@ -771,9 +901,9 @@ virtio_recv_mergeable_pkts(void *rx_queue, */ uint16_t rcv_cnt = RTE_MIN(seg_res, RTE_DIM(rcv_pkts)); - if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) { + if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { uint32_t rx_num = - virtqueue_dequeue_burst_rx(rxvq, + virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, rcv_cnt); i += rx_num; rcv_cnt = rx_num; @@ -781,8 +911,8 @@ virtio_recv_mergeable_pkts(void *rx_queue, PMD_RX_LOG(ERR, "No enough segments for packet."); nb_enqueued++; - virtio_discard_rxbuf(rxvq, rxm); - rxvq->errors++; + virtio_discard_rxbuf(vq, rxm); + rxvq->stats.errors++; break; } @@ -792,7 +922,6 @@ virtio_recv_mergeable_pkts(void *rx_queue, rxm = rcv_pkts[extra_idx]; rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; - rxm->next = NULL; rxm->pkt_len = (uint32_t)(len[extra_idx]); rxm->data_len = (uint16_t)(len[extra_idx]); @@ -812,24 +941,24 @@ virtio_recv_mergeable_pkts(void *rx_queue, VIRTIO_DUMP_PACKET(rx_pkts[nb_rx], rx_pkts[nb_rx]->data_len); - rxvq->bytes += rx_pkts[nb_rx]->pkt_len; - virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]); + rxvq->stats.bytes += rx_pkts[nb_rx]->pkt_len; + virtio_update_packet_stats(&rxvq->stats, rx_pkts[nb_rx]); nb_rx++; } - rxvq->packets += nb_rx; + rxvq->stats.packets += nb_rx; /* Allocate new mbuf for the used descriptor */ error = ENOSPC; - while (likely(!virtqueue_full(rxvq))) { - new_mbuf = rte_rxmbuf_alloc(rxvq->mpool); + while (likely(!virtqueue_full(vq))) { + new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool); if (unlikely(new_mbuf == NULL)) { struct rte_eth_dev *dev = &rte_eth_devices[rxvq->port_id]; dev->data->rx_mbuf_alloc_failed++; break; } - error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf); + error = virtqueue_enqueue_recv_refill(vq, new_mbuf); if (unlikely(error)) { rte_pktmbuf_free(new_mbuf); break; @@ -838,10 +967,10 @@ virtio_recv_mergeable_pkts(void *rx_queue, } if (likely(nb_enqueued)) { - vq_update_avail_idx(rxvq); + vq_update_avail_idx(vq); - if (unlikely(virtqueue_kick_prepare(rxvq))) { - virtqueue_notify(rxvq); + if (unlikely(virtqueue_kick_prepare(vq))) { + virtqueue_notify(vq); PMD_RX_LOG(DEBUG, "Notified"); } } @@ -852,21 +981,25 @@ virtio_recv_mergeable_pkts(void *rx_queue, uint16_t virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { - struct virtqueue *txvq = tx_queue; - struct virtio_hw *hw = txvq->hw; + struct virtnet_tx *txvq = tx_queue; + struct virtqueue *vq = txvq->vq; + struct virtio_hw *hw = vq->hw; uint16_t hdr_size = hw->vtnet_hdr_size; - uint16_t nb_used, nb_tx; + uint16_t nb_used, nb_tx = 0; int error; + if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) + return nb_tx; + if (unlikely(nb_pkts < 1)) return nb_pkts; PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); - nb_used = VIRTQUEUE_NUSED(txvq); + nb_used = VIRTQUEUE_NUSED(vq); virtio_rmb(); - if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh)) - virtio_xmit_cleanup(txvq, nb_used); + if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) + virtio_xmit_cleanup(vq, nb_used); for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { struct rte_mbuf *txm = tx_pkts[nb_tx]; @@ -882,8 +1015,10 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) } /* optimize ring usage */ - if (vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) && + if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || + vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && rte_mbuf_refcnt_read(txm) == 1 && + RTE_MBUF_DIRECT(txm) && txm->nb_segs == 1 && rte_pktmbuf_headroom(txm) >= hdr_size && rte_is_aligned(rte_pktmbuf_mtod(txm, char *), @@ -899,16 +1034,16 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) * default => number of segments + 1 */ slots = use_indirect ? 1 : (txm->nb_segs + !can_push); - need = slots - txvq->vq_free_cnt; + need = slots - vq->vq_free_cnt; /* Positive value indicates it need free vring descriptors */ if (unlikely(need > 0)) { - nb_used = VIRTQUEUE_NUSED(txvq); + nb_used = VIRTQUEUE_NUSED(vq); virtio_rmb(); need = RTE_MIN(need, (int)nb_used); - virtio_xmit_cleanup(txvq, need); - need = slots - txvq->vq_free_cnt; + virtio_xmit_cleanup(vq, need); + need = slots - vq->vq_free_cnt; if (unlikely(need > 0)) { PMD_TX_LOG(ERR, "No free tx descriptors to transmit"); @@ -919,17 +1054,17 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* Enqueue Packet buffers */ virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, can_push); - txvq->bytes += txm->pkt_len; - virtio_update_packet_stats(txvq, txm); + txvq->stats.bytes += txm->pkt_len; + virtio_update_packet_stats(&txvq->stats, txm); } - txvq->packets += nb_tx; + txvq->stats.packets += nb_tx; if (likely(nb_tx)) { - vq_update_avail_idx(txvq); + vq_update_avail_idx(vq); - if (unlikely(virtqueue_kick_prepare(txvq))) { - virtqueue_notify(txvq); + if (unlikely(virtqueue_kick_prepare(vq))) { + virtqueue_notify(vq); PMD_TX_LOG(DEBUG, "Notified backend after xmit"); } }