X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_pmd_virtio%2Fvirtio_rxtx.c;h=29c9ceada53edd98ee5bf49162004287bf2246c5;hb=08b563ffb19d8baf59dd84200f25bc85031d18a7;hp=4c45cdbb6f2a9e5f81192390cea0bf0524995c7d;hpb=403ac76b37833dad5e2b65403f35428919dcc8ee;p=dpdk.git diff --git a/lib/librte_pmd_virtio/virtio_rxtx.c b/lib/librte_pmd_virtio/virtio_rxtx.c index 4c45cdbb6f..29c9ceada5 100644 --- a/lib/librte_pmd_virtio/virtio_rxtx.c +++ b/lib/librte_pmd_virtio/virtio_rxtx.c @@ -118,7 +118,7 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, } rte_prefetch0(cookie); - rte_packet_prefetch(cookie->pkt.data); + rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); rx_pkts[i] = cookie; vq->vq_used_cons_idx++; vq_ring_free_chain(vq, desc_idx); @@ -146,6 +146,7 @@ static inline int virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) { struct vq_desc_extra *dxp; + struct virtio_hw *hw = vq->hw; struct vring_desc *start_dp; uint16_t needed = 1; uint16_t head_idx, idx; @@ -165,9 +166,11 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) dxp->ndescs = needed; start_dp = vq->vq_ring.desc; - start_dp[idx].addr = - (uint64_t) (cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM - sizeof(struct virtio_net_hdr)); - start_dp[idx].len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + sizeof(struct virtio_net_hdr); + start_dp[idx].addr = + (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM + - hw->vtnet_hdr_size); + start_dp[idx].len = + cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; start_dp[idx].flags = VRING_DESC_F_WRITE; idx = start_dp[idx].next; vq->vq_desc_head_idx = idx; @@ -184,8 +187,10 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) { struct vq_desc_extra *dxp; struct vring_desc *start_dp; - uint16_t needed = 2; + uint16_t seg_num = cookie->nb_segs; + uint16_t needed = 1 + seg_num; uint16_t head_idx, idx; + uint16_t head_size = txvq->hw->vtnet_hdr_size; if (unlikely(txvq->vq_free_cnt == 0)) return -ENOSPC; @@ -198,19 +203,25 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) idx = head_idx; dxp = &txvq->vq_descx[idx]; if (dxp->cookie != NULL) - rte_pktmbuf_free_seg(dxp->cookie); + rte_pktmbuf_free(dxp->cookie); dxp->cookie = (void *)cookie; dxp->ndescs = needed; start_dp = txvq->vq_ring.desc; - start_dp[idx].addr = - txvq->virtio_net_hdr_mem + idx * sizeof(struct virtio_net_hdr); - start_dp[idx].len = sizeof(struct virtio_net_hdr); + start_dp[idx].addr = + txvq->virtio_net_hdr_mem + idx * head_size; + start_dp[idx].len = (uint32_t)head_size; start_dp[idx].flags = VRING_DESC_F_NEXT; - idx = start_dp[idx].next; - start_dp[idx].addr = RTE_MBUF_DATA_DMA_ADDR(cookie); - start_dp[idx].len = cookie->pkt.data_len; - start_dp[idx].flags = 0; + + for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) { + idx = start_dp[idx].next; + start_dp[idx].addr = RTE_MBUF_DATA_DMA_ADDR(cookie); + start_dp[idx].len = cookie->data_len; + start_dp[idx].flags = VRING_DESC_F_NEXT; + cookie = cookie->next; + } + + start_dp[idx].flags &= ~VRING_DESC_F_NEXT; idx = start_dp[idx].next; txvq->vq_desc_head_idx = idx; if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) @@ -227,19 +238,19 @@ rte_rxmbuf_alloc(struct rte_mempool *mp) struct rte_mbuf *m; m = __rte_mbuf_raw_alloc(mp); - __rte_mbuf_sanity_check_raw(m, RTE_MBUF_PKT, 0); + __rte_mbuf_sanity_check_raw(m, 0); return m; } static void -virtio_dev_vring_start(struct rte_eth_dev *dev, struct virtqueue *vq, int queue_type) +virtio_dev_vring_start(struct virtqueue *vq, int queue_type) { struct rte_mbuf *m; int i, nbufs, error, size = vq->vq_nentries; struct vring *vr = &vq->vq_ring; uint8_t *ring_mem = vq->vq_ring_virt_mem; - char vq_name[VIRTQUEUE_MAX_NAME_SZ]; + PMD_INIT_FUNC_TRACE(); /* @@ -264,10 +275,6 @@ virtio_dev_vring_start(struct rte_eth_dev *dev, struct virtqueue *vq, int queue_ */ virtqueue_disable_intr(vq); - snprintf(vq_name, sizeof(vq_name), "port_%d_rx_vq", - dev->data->port_id); - PMD_INIT_LOG(DEBUG, "vq name: %s", vq->vq_name); - /* Only rx virtqueue needs mbufs to be allocated at initialization */ if (queue_type == VTNET_RQ) { if (vq->mpool == NULL) @@ -288,7 +295,7 @@ virtio_dev_vring_start(struct rte_eth_dev *dev, struct virtqueue *vq, int queue_ error = virtqueue_enqueue_recv_refill(vq, m); if (error) { - rte_pktmbuf_free_seg(m); + rte_pktmbuf_free(m); break; } nbufs++; @@ -321,7 +328,7 @@ virtio_dev_cq_start(struct rte_eth_dev *dev) struct virtio_hw *hw = VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private); - virtio_dev_vring_start(dev, hw->cvq, VTNET_CQ); + virtio_dev_vring_start(hw->cvq, VTNET_CQ); VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq); } @@ -341,13 +348,13 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev) /* Start rx vring. */ for (i = 0; i < dev->data->nb_rx_queues; i++) { - virtio_dev_vring_start(dev, dev->data->rx_queues[i], VTNET_RQ); + virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ); VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); } /* Start tx vring. */ for (i = 0; i < dev->data->nb_tx_queues; i++) { - virtio_dev_vring_start(dev, dev->data->tx_queues[i], VTNET_TQ); + virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ); VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); } } @@ -391,13 +398,20 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, - __rte_unused const struct rte_eth_txconf *tx_conf) + const struct rte_eth_txconf *tx_conf) { uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; struct virtqueue *vq; int ret; PMD_INIT_FUNC_TRACE(); + + if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOOFFLOADS) + != ETH_TXQ_FLAGS_NOOFFLOADS) { + PMD_INIT_LOG(ERR, "TX checksum offload not supported\n"); + return -EINVAL; + } + ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx, nb_desc, socket_id, &vq); if (ret < 0) { @@ -420,7 +434,7 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) error = virtqueue_enqueue_recv_refill(vq, m); if (unlikely(error)) { RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf"); - rte_pktmbuf_free_seg(m); + rte_pktmbuf_free(m); } } @@ -430,13 +444,13 @@ uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct virtqueue *rxvq = rx_queue; - struct virtio_hw *hw = rxvq->hw; struct rte_mbuf *rxm, *new_mbuf; uint16_t nb_used, num, nb_rx = 0; uint32_t len[VIRTIO_MBUF_BURST_SZ]; struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; int error; uint32_t i, nb_enqueued = 0; + const uint32_t hdr_size = sizeof(struct virtio_net_hdr); nb_used = VIRTQUEUE_NUSED(rxvq); @@ -457,8 +471,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); - if (unlikely(len[i] - < (uint32_t)hw->vtnet_hdr_size + ETHER_HDR_LEN)) { + if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) { PMD_RX_LOG(ERR, "Packet drop"); nb_enqueued++; virtio_discard_rxbuf(rxvq, rxm); @@ -466,19 +479,169 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) continue; } - rxm->pkt.in_port = rxvq->port_id; - rxm->pkt.data = (char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM; - rxm->pkt.nb_segs = 1; - rxm->pkt.next = NULL; - rxm->pkt.pkt_len = (uint32_t)(len[i] - - sizeof(struct virtio_net_hdr)); - rxm->pkt.data_len = (uint16_t)(len[i] - - sizeof(struct virtio_net_hdr)); + rxm->port = rxvq->port_id; + rxm->data_off = RTE_PKTMBUF_HEADROOM; - VIRTIO_DUMP_PACKET(rxm, rxm->pkt.data_len); + rxm->nb_segs = 1; + rxm->next = NULL; + rxm->pkt_len = (uint32_t)(len[i] - hdr_size); + rxm->data_len = (uint16_t)(len[i] - hdr_size); + + VIRTIO_DUMP_PACKET(rxm, rxm->data_len); rx_pkts[nb_rx++] = rxm; - rxvq->bytes += len[i] - sizeof(struct virtio_net_hdr); + rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len; + } + + rxvq->packets += nb_rx; + + /* Allocate new mbuf for the used descriptor */ + error = ENOSPC; + while (likely(!virtqueue_full(rxvq))) { + new_mbuf = rte_rxmbuf_alloc(rxvq->mpool); + if (unlikely(new_mbuf == NULL)) { + struct rte_eth_dev *dev + = &rte_eth_devices[rxvq->port_id]; + dev->data->rx_mbuf_alloc_failed++; + break; + } + error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf); + if (unlikely(error)) { + rte_pktmbuf_free(new_mbuf); + break; + } + nb_enqueued++; + } + + if (likely(nb_enqueued)) { + if (unlikely(virtqueue_kick_prepare(rxvq))) { + virtqueue_notify(rxvq); + PMD_RX_LOG(DEBUG, "Notified\n"); + } + } + + vq_update_avail_idx(rxvq); + + return nb_rx; +} + +uint16_t +virtio_recv_mergeable_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct virtqueue *rxvq = rx_queue; + struct rte_mbuf *rxm, *new_mbuf; + uint16_t nb_used, num, nb_rx = 0; + uint32_t len[VIRTIO_MBUF_BURST_SZ]; + struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; + struct rte_mbuf *prev; + int error; + uint32_t i = 0, nb_enqueued = 0; + uint32_t seg_num = 0; + uint16_t extra_idx = 0; + uint32_t seg_res = 0; + const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); + + nb_used = VIRTQUEUE_NUSED(rxvq); + + rmb(); + + if (nb_used == 0) + return 0; + + PMD_RX_LOG(DEBUG, "used:%d\n", nb_used); + + while (i < nb_used) { + struct virtio_net_hdr_mrg_rxbuf *header; + + if (nb_rx == nb_pkts) + break; + + num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1); + if (num != 1) + continue; + + i++; + + PMD_RX_LOG(DEBUG, "dequeue:%d\n", num); + PMD_RX_LOG(DEBUG, "packet len:%d\n", len[0]); + + rxm = rcv_pkts[0]; + + if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) { + PMD_RX_LOG(ERR, "Packet drop\n"); + nb_enqueued++; + virtio_discard_rxbuf(rxvq, rxm); + rxvq->errors++; + continue; + } + + header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr + + RTE_PKTMBUF_HEADROOM - hdr_size); + seg_num = header->num_buffers; + + if (seg_num == 0) + seg_num = 1; + + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rxm->nb_segs = seg_num; + rxm->next = NULL; + rxm->pkt_len = (uint32_t)(len[0] - hdr_size); + rxm->data_len = (uint16_t)(len[0] - hdr_size); + + rxm->port = rxvq->port_id; + rx_pkts[nb_rx] = rxm; + prev = rxm; + + seg_res = seg_num - 1; + + while (seg_res != 0) { + /* + * Get extra segments for current uncompleted packet. + */ + uint32_t rcv_cnt = + RTE_MIN(seg_res, RTE_DIM(rcv_pkts)); + if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) { + uint32_t rx_num = + virtqueue_dequeue_burst_rx(rxvq, + rcv_pkts, len, rcv_cnt); + i += rx_num; + rcv_cnt = rx_num; + } else { + PMD_RX_LOG(ERR, + "No enough segments for packet.\n"); + nb_enqueued++; + virtio_discard_rxbuf(rxvq, rxm); + rxvq->errors++; + break; + } + + extra_idx = 0; + + while (extra_idx < rcv_cnt) { + rxm = rcv_pkts[extra_idx]; + + rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; + rxm->next = NULL; + rxm->pkt_len = (uint32_t)(len[extra_idx]); + rxm->data_len = (uint16_t)(len[extra_idx]); + + if (prev) + prev->next = rxm; + + prev = rxm; + rx_pkts[nb_rx]->pkt_len += rxm->pkt_len; + extra_idx++; + }; + seg_res -= rcv_cnt; + } + + VIRTIO_DUMP_PACKET(rx_pkts[nb_rx], + rx_pkts[nb_rx]->data_len); + + rxvq->bytes += rx_pkts[nb_rx]->pkt_len; + nb_rx++; } rxvq->packets += nb_rx; @@ -495,11 +658,12 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) } error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf); if (unlikely(error)) { - rte_pktmbuf_free_seg(new_mbuf); + rte_pktmbuf_free(new_mbuf); break; } nb_enqueued++; } + if (likely(nb_enqueued)) { if (unlikely(virtqueue_kick_prepare(rxvq))) { virtqueue_notify(rxvq); @@ -533,12 +697,16 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) num = (uint16_t)(likely(nb_used < VIRTIO_MBUF_BURST_SZ) ? nb_used : VIRTIO_MBUF_BURST_SZ); while (nb_tx < nb_pkts) { - if (virtqueue_full(txvq) && num) { + int need = tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt; + int deq_cnt = RTE_MIN(need, (int)num); + + num -= (deq_cnt > 0) ? deq_cnt : 0; + while (deq_cnt > 0) { virtqueue_dequeue_pkt_tx(txvq); - num--; + deq_cnt--; } - if (!virtqueue_full(txvq)) { + if (tx_pkts[nb_tx]->nb_segs <= txvq->vq_free_cnt) { txm = tx_pkts[nb_tx]; /* Enqueue Packet buffers */ error = virtqueue_enqueue_xmit(txvq, txm); @@ -552,7 +720,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) break; } nb_tx++; - txvq->bytes += txm->pkt.data_len; + txvq->bytes += txm->pkt_len; } else { PMD_TX_LOG(ERR, "No free tx descriptors to transmit"); break;