1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
11 #include <rte_cycles.h>
12 #include <rte_memory.h>
13 #include <rte_branch_prediction.h>
14 #include <rte_mempool.h>
15 #include <rte_malloc.h>
17 #include <rte_ether.h>
18 #include <ethdev_driver.h>
19 #include <rte_prefetch.h>
20 #include <rte_string_fns.h>
21 #include <rte_errno.h>
22 #include <rte_byteorder.h>
28 #include "virtio_logs.h"
29 #include "virtio_ethdev.h"
31 #include "virtqueue.h"
32 #include "virtio_rxtx.h"
33 #include "virtio_rxtx_simple.h"
34 #include "virtio_ring.h"
36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0)
43 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num)
45 vq->vq_free_cnt += num;
46 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1);
50 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
52 struct vring_desc *dp, *dp_tail;
53 struct vq_desc_extra *dxp;
54 uint16_t desc_idx_last = desc_idx;
56 dp = &vq->vq_split.ring.desc[desc_idx];
57 dxp = &vq->vq_descx[desc_idx];
58 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
59 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
60 while (dp->flags & VRING_DESC_F_NEXT) {
61 desc_idx_last = dp->next;
62 dp = &vq->vq_split.ring.desc[dp->next];
68 * We must append the existing free chain, if any, to the end of
69 * newly freed chain. If the virtqueue was completely used, then
70 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
72 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
73 vq->vq_desc_head_idx = desc_idx;
75 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx];
76 dp_tail->next = desc_idx;
79 vq->vq_desc_tail_idx = desc_idx_last;
80 dp->next = VQ_RING_DESC_CHAIN_END;
84 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
86 uint32_t s = mbuf->pkt_len;
87 struct rte_ether_addr *ea;
92 stats->size_bins[1]++;
93 } else if (s > 64 && s < 1024) {
96 /* count zeros, and offset into correct bin */
97 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
98 stats->size_bins[bin]++;
101 stats->size_bins[0]++;
103 stats->size_bins[6]++;
105 stats->size_bins[7]++;
108 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *);
109 if (rte_is_multicast_ether_addr(ea)) {
110 if (rte_is_broadcast_ether_addr(ea))
118 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m)
120 VIRTIO_DUMP_PACKET(m, m->data_len);
122 virtio_update_packet_stats(&rxvq->stats, m);
126 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
127 struct rte_mbuf **rx_pkts,
131 struct rte_mbuf *cookie;
134 struct vring_packed_desc *desc;
137 desc = vq->vq_packed.ring.desc;
139 for (i = 0; i < num; i++) {
140 used_idx = vq->vq_used_cons_idx;
141 /* desc_is_used has a load-acquire or rte_io_rmb inside
142 * and wait for used desc in virtqueue.
144 if (!desc_is_used(&desc[used_idx], vq))
146 len[i] = desc[used_idx].len;
147 id = desc[used_idx].id;
148 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
149 if (unlikely(cookie == NULL)) {
150 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
151 vq->vq_used_cons_idx);
154 rte_prefetch0(cookie);
155 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
159 vq->vq_used_cons_idx++;
160 if (vq->vq_used_cons_idx >= vq->vq_nentries) {
161 vq->vq_used_cons_idx -= vq->vq_nentries;
162 vq->vq_packed.used_wrap_counter ^= 1;
170 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
171 uint32_t *len, uint16_t num)
173 struct vring_used_elem *uep;
174 struct rte_mbuf *cookie;
175 uint16_t used_idx, desc_idx;
178 /* Caller does the check */
179 for (i = 0; i < num ; i++) {
180 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
181 uep = &vq->vq_split.ring.used->ring[used_idx];
182 desc_idx = (uint16_t) uep->id;
184 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
186 if (unlikely(cookie == NULL)) {
187 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
188 vq->vq_used_cons_idx);
192 rte_prefetch0(cookie);
193 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
195 vq->vq_used_cons_idx++;
196 vq_ring_free_chain(vq, desc_idx);
197 vq->vq_descx[desc_idx].cookie = NULL;
204 virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
205 struct rte_mbuf **rx_pkts,
209 struct vring_used_elem *uep;
210 struct rte_mbuf *cookie;
211 uint16_t used_idx = 0;
214 if (unlikely(num == 0))
217 for (i = 0; i < num; i++) {
218 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);
219 /* Desc idx same as used idx */
220 uep = &vq->vq_split.ring.used->ring[used_idx];
222 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie;
224 if (unlikely(cookie == NULL)) {
225 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
226 vq->vq_used_cons_idx);
230 rte_prefetch0(cookie);
231 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
233 vq->vq_used_cons_idx++;
234 vq->vq_descx[used_idx].cookie = NULL;
237 vq_ring_free_inorder(vq, used_idx, i);
242 virtqueue_enqueue_refill_inorder(struct virtqueue *vq,
243 struct rte_mbuf **cookies,
246 struct vq_desc_extra *dxp;
247 struct virtio_hw *hw = vq->hw;
248 struct vring_desc *start_dp;
249 uint16_t head_idx, idx, i = 0;
251 if (unlikely(vq->vq_free_cnt == 0))
253 if (unlikely(vq->vq_free_cnt < num))
256 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1);
257 start_dp = vq->vq_split.ring.desc;
260 idx = head_idx & (vq->vq_nentries - 1);
261 dxp = &vq->vq_descx[idx];
262 dxp->cookie = (void *)cookies[i];
265 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookies[i], vq) +
266 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
267 start_dp[idx].len = cookies[i]->buf_len -
268 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
269 start_dp[idx].flags = VRING_DESC_F_WRITE;
271 vq_update_avail_ring(vq, idx);
276 vq->vq_desc_head_idx += num;
277 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
282 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie,
285 struct vq_desc_extra *dxp;
286 struct virtio_hw *hw = vq->hw;
287 struct vring_desc *start_dp = vq->vq_split.ring.desc;
290 if (unlikely(vq->vq_free_cnt == 0))
292 if (unlikely(vq->vq_free_cnt < num))
295 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries))
298 for (i = 0; i < num; i++) {
299 idx = vq->vq_desc_head_idx;
300 dxp = &vq->vq_descx[idx];
301 dxp->cookie = (void *)cookie[i];
304 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) +
305 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
306 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM +
308 start_dp[idx].flags = VRING_DESC_F_WRITE;
309 vq->vq_desc_head_idx = start_dp[idx].next;
310 vq_update_avail_ring(vq, idx);
311 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) {
312 vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
317 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
323 virtqueue_refill_single_packed(struct virtqueue *vq,
324 struct vring_packed_desc *dp,
325 struct rte_mbuf *cookie)
327 uint16_t flags = vq->vq_packed.cached_flags;
328 struct virtio_hw *hw = vq->hw;
330 dp->addr = VIRTIO_MBUF_ADDR(cookie, vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
331 dp->len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
333 virtqueue_store_flags_packed(dp, flags, hw->weak_barriers);
335 if (++vq->vq_avail_idx >= vq->vq_nentries) {
336 vq->vq_avail_idx -= vq->vq_nentries;
337 vq->vq_packed.cached_flags ^=
338 VRING_PACKED_DESC_F_AVAIL_USED;
339 flags = vq->vq_packed.cached_flags;
344 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq,
345 struct rte_mbuf **cookie, uint16_t num)
347 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
348 struct vq_desc_extra *dxp;
352 if (unlikely(vq->vq_free_cnt == 0))
354 if (unlikely(vq->vq_free_cnt < num))
357 for (i = 0; i < num; i++) {
358 idx = vq->vq_avail_idx;
359 dxp = &vq->vq_descx[idx];
360 dxp->cookie = (void *)cookie[i];
363 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]);
365 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
370 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
371 struct rte_mbuf **cookie, uint16_t num)
373 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
374 struct vq_desc_extra *dxp;
378 if (unlikely(vq->vq_free_cnt == 0))
380 if (unlikely(vq->vq_free_cnt < num))
383 for (i = 0; i < num; i++) {
384 idx = vq->vq_avail_idx;
385 did = start_dp[idx].id;
386 dxp = &vq->vq_descx[did];
387 dxp->cookie = (void *)cookie[i];
390 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]);
392 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
396 /* When doing TSO, the IP length is not included in the pseudo header
397 * checksum of the packet given to the PMD, but for virtio it is
401 virtio_tso_fix_cksum(struct rte_mbuf *m)
403 /* common case: header is not fragmented */
404 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len +
406 struct rte_ipv4_hdr *iph;
407 struct rte_ipv6_hdr *ip6h;
408 struct rte_tcp_hdr *th;
409 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen;
412 iph = rte_pktmbuf_mtod_offset(m,
413 struct rte_ipv4_hdr *, m->l2_len);
414 th = RTE_PTR_ADD(iph, m->l3_len);
415 if ((iph->version_ihl >> 4) == 4) {
416 iph->hdr_checksum = 0;
417 iph->hdr_checksum = rte_ipv4_cksum(iph);
418 ip_len = iph->total_length;
419 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) -
422 ip6h = (struct rte_ipv6_hdr *)iph;
423 ip_paylen = ip6h->payload_len;
426 /* calculate the new phdr checksum not including ip_paylen */
427 prev_cksum = th->cksum;
430 tmp = (tmp & 0xffff) + (tmp >> 16);
433 /* replace it in the packet */
434 th->cksum = new_cksum;
442 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq,
443 struct rte_mbuf **cookies,
446 struct vq_desc_extra *dxp;
447 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
448 struct vring_desc *start_dp;
449 struct virtio_net_hdr *hdr;
451 int16_t head_size = vq->hw->vtnet_hdr_size;
454 idx = vq->vq_desc_head_idx;
455 start_dp = vq->vq_split.ring.desc;
458 idx = idx & (vq->vq_nentries - 1);
459 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
460 dxp->cookie = (void *)cookies[i];
462 virtio_update_packet_stats(&txvq->stats, cookies[i]);
464 hdr = rte_pktmbuf_mtod_offset(cookies[i],
465 struct virtio_net_hdr *, -head_size);
467 /* if offload disabled, hdr is not zeroed yet, do it now */
468 if (!vq->hw->has_tx_offload)
469 virtqueue_clear_net_hdr(hdr);
471 virtqueue_xmit_offload(hdr, cookies[i]);
473 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size;
474 start_dp[idx].len = cookies[i]->data_len + head_size;
475 start_dp[idx].flags = 0;
478 vq_update_avail_ring(vq, idx);
484 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
485 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1);
489 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
490 struct rte_mbuf *cookie,
493 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
494 struct vring_packed_desc *dp;
495 struct vq_desc_extra *dxp;
496 uint16_t idx, id, flags;
497 int16_t head_size = vq->hw->vtnet_hdr_size;
498 struct virtio_net_hdr *hdr;
500 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
501 idx = vq->vq_avail_idx;
502 dp = &vq->vq_packed.ring.desc[idx];
504 dxp = &vq->vq_descx[id];
506 dxp->cookie = cookie;
508 flags = vq->vq_packed.cached_flags;
510 /* prepend cannot fail, checked by caller */
511 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
514 /* if offload disabled, hdr is not zeroed yet, do it now */
515 if (!vq->hw->has_tx_offload)
516 virtqueue_clear_net_hdr(hdr);
518 virtqueue_xmit_offload(hdr, cookie);
520 dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size;
521 dp->len = cookie->data_len + head_size;
524 if (++vq->vq_avail_idx >= vq->vq_nentries) {
525 vq->vq_avail_idx -= vq->vq_nentries;
526 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
532 vq->vq_desc_head_idx = dxp->next;
533 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
534 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
537 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
541 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
542 uint16_t needed, int use_indirect, int can_push,
545 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
546 struct vq_desc_extra *dxp;
547 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
548 struct vring_desc *start_dp;
549 uint16_t seg_num = cookie->nb_segs;
550 uint16_t head_idx, idx;
551 int16_t head_size = vq->hw->vtnet_hdr_size;
552 bool prepend_header = false;
553 struct virtio_net_hdr *hdr;
555 head_idx = vq->vq_desc_head_idx;
558 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
560 dxp = &vq->vq_descx[idx];
561 dxp->cookie = (void *)cookie;
562 dxp->ndescs = needed;
564 start_dp = vq->vq_split.ring.desc;
567 /* prepend cannot fail, checked by caller */
568 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
570 prepend_header = true;
572 /* if offload disabled, it is not zeroed below, do it now */
573 if (!vq->hw->has_tx_offload)
574 virtqueue_clear_net_hdr(hdr);
575 } else if (use_indirect) {
576 /* setup tx ring slot to point to indirect
577 * descriptor list stored in reserved region.
579 * the first slot in indirect ring is already preset
580 * to point to the header in reserved region
582 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
583 RTE_PTR_DIFF(&txr[idx].tx_indir, txr);
584 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc);
585 start_dp[idx].flags = VRING_DESC_F_INDIRECT;
586 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
588 /* loop below will fill in rest of the indirect elements */
589 start_dp = txr[idx].tx_indir;
592 /* setup first tx ring slot to point to header
593 * stored in reserved region.
595 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
596 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
597 start_dp[idx].len = vq->hw->vtnet_hdr_size;
598 start_dp[idx].flags = VRING_DESC_F_NEXT;
599 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
601 idx = start_dp[idx].next;
604 if (vq->hw->has_tx_offload)
605 virtqueue_xmit_offload(hdr, cookie);
608 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);
609 start_dp[idx].len = cookie->data_len;
610 if (prepend_header) {
611 start_dp[idx].addr -= head_size;
612 start_dp[idx].len += head_size;
613 prepend_header = false;
615 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0;
616 idx = start_dp[idx].next;
617 } while ((cookie = cookie->next) != NULL);
620 idx = vq->vq_split.ring.desc[head_idx].next;
622 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
624 vq->vq_desc_head_idx = idx;
625 vq_update_avail_ring(vq, head_idx);
628 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
629 vq->vq_desc_tail_idx = idx;
634 virtio_dev_cq_start(struct rte_eth_dev *dev)
636 struct virtio_hw *hw = dev->data->dev_private;
639 rte_spinlock_init(&hw->cvq->lock);
640 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq));
645 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
648 unsigned int socket_id __rte_unused,
649 const struct rte_eth_rxconf *rx_conf,
650 struct rte_mempool *mp)
652 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
653 struct virtio_hw *hw = dev->data->dev_private;
654 struct virtqueue *vq = hw->vqs[vq_idx];
655 struct virtnet_rx *rxvq;
656 uint16_t rx_free_thresh;
660 PMD_INIT_FUNC_TRACE();
662 if (rx_conf->rx_deferred_start) {
663 PMD_INIT_LOG(ERR, "Rx deferred start is not supported");
667 buf_size = virtio_rx_mem_pool_buf_size(mp);
668 if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size,
669 hw->rx_ol_scatter, &error)) {
670 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s",
675 rx_free_thresh = rx_conf->rx_free_thresh;
676 if (rx_free_thresh == 0)
678 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH);
680 if (rx_free_thresh & 0x3) {
681 PMD_INIT_LOG(ERR, "rx_free_thresh must be multiples of four."
682 " (rx_free_thresh=%u port=%u queue=%u)",
683 rx_free_thresh, dev->data->port_id, queue_idx);
687 if (rx_free_thresh >= vq->vq_nentries) {
688 PMD_INIT_LOG(ERR, "rx_free_thresh must be less than the "
689 "number of RX entries (%u)."
690 " (rx_free_thresh=%u port=%u queue=%u)",
692 rx_free_thresh, dev->data->port_id, queue_idx);
695 vq->vq_free_thresh = rx_free_thresh;
698 * For split ring vectorized path descriptors number must be
699 * equal to the ring size.
701 if (nb_desc > vq->vq_nentries ||
702 (!virtio_with_packed_queue(hw) && hw->use_vec_rx)) {
703 nb_desc = vq->vq_nentries;
705 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
708 rxvq->queue_id = queue_idx;
710 dev->data->rx_queues[queue_idx] = rxvq;
716 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
718 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
719 struct virtio_hw *hw = dev->data->dev_private;
720 struct virtqueue *vq = hw->vqs[vq_idx];
721 struct virtnet_rx *rxvq = &vq->rxq;
725 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER);
727 PMD_INIT_FUNC_TRACE();
729 /* Allocate blank mbufs for the each rx descriptor */
732 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) {
733 for (desc_idx = 0; desc_idx < vq->vq_nentries;
735 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx;
736 vq->vq_split.ring.desc[desc_idx].flags =
740 virtio_rxq_vec_setup(rxvq);
743 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf));
744 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++)
745 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf;
747 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) {
748 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
749 virtio_rxq_rearm_vec(rxvq);
750 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
752 } else if (!virtio_with_packed_queue(vq->hw) && in_order) {
753 if ((!virtqueue_full(vq))) {
754 uint16_t free_cnt = vq->vq_free_cnt;
755 struct rte_mbuf *pkts[free_cnt];
757 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts,
759 error = virtqueue_enqueue_refill_inorder(vq,
762 if (unlikely(error)) {
763 for (i = 0; i < free_cnt; i++)
764 rte_pktmbuf_free(pkts[i]);
770 vq_update_avail_idx(vq);
773 while (!virtqueue_full(vq)) {
774 m = rte_mbuf_raw_alloc(rxvq->mpool);
778 /* Enqueue allocated buffers */
779 if (virtio_with_packed_queue(vq->hw))
780 error = virtqueue_enqueue_recv_refill_packed_init(vq,
783 error = virtqueue_enqueue_recv_refill(vq,
792 if (!virtio_with_packed_queue(vq->hw))
793 vq_update_avail_idx(vq);
796 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
804 * struct rte_eth_dev *dev: Used to update dev
805 * uint16_t nb_desc: Defaults to values read from config space
806 * unsigned int socket_id: Used to allocate memzone
807 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
808 * uint16_t queue_idx: Just used as an index in dev txq list
811 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
814 unsigned int socket_id __rte_unused,
815 const struct rte_eth_txconf *tx_conf)
817 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
818 struct virtio_hw *hw = dev->data->dev_private;
819 struct virtqueue *vq = hw->vqs[vq_idx];
820 struct virtnet_tx *txvq;
821 uint16_t tx_free_thresh;
823 PMD_INIT_FUNC_TRACE();
825 if (tx_conf->tx_deferred_start) {
826 PMD_INIT_LOG(ERR, "Tx deferred start is not supported");
830 if (nb_desc == 0 || nb_desc > vq->vq_nentries)
831 nb_desc = vq->vq_nentries;
832 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
835 txvq->queue_id = queue_idx;
837 tx_free_thresh = tx_conf->tx_free_thresh;
838 if (tx_free_thresh == 0)
840 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
842 if (tx_free_thresh >= (vq->vq_nentries - 3)) {
843 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the "
844 "number of TX entries minus 3 (%u)."
845 " (tx_free_thresh=%u port=%u queue=%u)",
847 tx_free_thresh, dev->data->port_id, queue_idx);
851 vq->vq_free_thresh = tx_free_thresh;
853 dev->data->tx_queues[queue_idx] = txvq;
858 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
861 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
862 struct virtio_hw *hw = dev->data->dev_private;
863 struct virtqueue *vq = hw->vqs[vq_idx];
865 PMD_INIT_FUNC_TRACE();
867 if (!virtio_with_packed_queue(hw)) {
868 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER))
869 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
878 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
882 * Requeue the discarded mbuf. This should always be
883 * successful since it was just dequeued.
885 if (virtio_with_packed_queue(vq->hw))
886 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1);
888 error = virtqueue_enqueue_recv_refill(vq, &m, 1);
890 if (unlikely(error)) {
891 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf");
897 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m)
901 error = virtqueue_enqueue_refill_inorder(vq, &m, 1);
902 if (unlikely(error)) {
903 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf");
908 /* Optionally fill offload information in structure */
910 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
912 struct rte_net_hdr_lens hdr_lens;
913 uint32_t hdrlen, ptype;
914 int l4_supported = 0;
917 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
920 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
922 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
923 m->packet_type = ptype;
924 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP ||
925 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP ||
926 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP)
929 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
930 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len;
931 if (hdr->csum_start <= hdrlen && l4_supported) {
932 m->ol_flags |= PKT_RX_L4_CKSUM_NONE;
934 /* Unknown proto or tunnel, do sw cksum. We can assume
935 * the cksum field is in the first segment since the
936 * buffers we provided to the host are large enough.
937 * In case of SCTP, this will be wrong since it's a CRC
938 * but there's nothing we can do.
940 uint16_t csum = 0, off;
942 if (rte_raw_cksum_mbuf(m, hdr->csum_start,
943 rte_pktmbuf_pkt_len(m) - hdr->csum_start,
946 if (likely(csum != 0xffff))
948 off = hdr->csum_offset + hdr->csum_start;
949 if (rte_pktmbuf_data_len(m) >= off + 1)
950 *rte_pktmbuf_mtod_offset(m, uint16_t *,
953 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) {
954 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
957 /* GSO request, save required information in mbuf */
958 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
959 /* Check unsupported modes */
960 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) ||
961 (hdr->gso_size == 0)) {
965 /* Update mss lengthes in mbuf */
966 m->tso_segsz = hdr->gso_size;
967 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
968 case VIRTIO_NET_HDR_GSO_TCPV4:
969 case VIRTIO_NET_HDR_GSO_TCPV6:
970 m->ol_flags |= PKT_RX_LRO | \
971 PKT_RX_L4_CKSUM_NONE;
981 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
983 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
985 struct virtnet_rx *rxvq = rx_queue;
986 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
987 struct virtio_hw *hw = vq->hw;
988 struct rte_mbuf *rxm;
989 uint16_t nb_used, num, nb_rx;
990 uint32_t len[VIRTIO_MBUF_BURST_SZ];
991 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
993 uint32_t i, nb_enqueued;
995 struct virtio_net_hdr *hdr;
998 if (unlikely(hw->started == 0))
1001 nb_used = virtqueue_nused(vq);
1003 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1004 if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1005 num = VIRTIO_MBUF_BURST_SZ;
1006 if (likely(num > DESC_PER_CACHELINE))
1007 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1009 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1010 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
1013 hdr_size = hw->vtnet_hdr_size;
1015 for (i = 0; i < num ; i++) {
1018 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1020 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1021 PMD_RX_LOG(ERR, "Packet drop");
1023 virtio_discard_rxbuf(vq, rxm);
1024 rxvq->stats.errors++;
1028 rxm->port = rxvq->port_id;
1029 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1033 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1034 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1036 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1037 RTE_PKTMBUF_HEADROOM - hdr_size);
1040 rte_vlan_strip(rxm);
1042 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1043 virtio_discard_rxbuf(vq, rxm);
1044 rxvq->stats.errors++;
1048 virtio_rx_stats_updated(rxvq, rxm);
1050 rx_pkts[nb_rx++] = rxm;
1053 rxvq->stats.packets += nb_rx;
1055 /* Allocate new mbuf for the used descriptor */
1056 if (likely(!virtqueue_full(vq))) {
1057 uint16_t free_cnt = vq->vq_free_cnt;
1058 struct rte_mbuf *new_pkts[free_cnt];
1060 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1062 error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1064 if (unlikely(error)) {
1065 for (i = 0; i < free_cnt; i++)
1066 rte_pktmbuf_free(new_pkts[i]);
1068 nb_enqueued += free_cnt;
1070 struct rte_eth_dev *dev =
1071 &rte_eth_devices[rxvq->port_id];
1072 dev->data->rx_mbuf_alloc_failed += free_cnt;
1076 if (likely(nb_enqueued)) {
1077 vq_update_avail_idx(vq);
1079 if (unlikely(virtqueue_kick_prepare(vq))) {
1080 virtqueue_notify(vq);
1081 PMD_RX_LOG(DEBUG, "Notified");
1089 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts,
1092 struct virtnet_rx *rxvq = rx_queue;
1093 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1094 struct virtio_hw *hw = vq->hw;
1095 struct rte_mbuf *rxm;
1096 uint16_t num, nb_rx;
1097 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1098 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1100 uint32_t i, nb_enqueued;
1102 struct virtio_net_hdr *hdr;
1105 if (unlikely(hw->started == 0))
1108 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts);
1109 if (likely(num > DESC_PER_CACHELINE))
1110 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1112 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1113 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1116 hdr_size = hw->vtnet_hdr_size;
1118 for (i = 0; i < num; i++) {
1121 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1123 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1124 PMD_RX_LOG(ERR, "Packet drop");
1126 virtio_discard_rxbuf(vq, rxm);
1127 rxvq->stats.errors++;
1131 rxm->port = rxvq->port_id;
1132 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1136 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1137 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1139 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1140 RTE_PKTMBUF_HEADROOM - hdr_size);
1143 rte_vlan_strip(rxm);
1145 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1146 virtio_discard_rxbuf(vq, rxm);
1147 rxvq->stats.errors++;
1151 virtio_rx_stats_updated(rxvq, rxm);
1153 rx_pkts[nb_rx++] = rxm;
1156 rxvq->stats.packets += nb_rx;
1158 /* Allocate new mbuf for the used descriptor */
1159 if (likely(!virtqueue_full(vq))) {
1160 uint16_t free_cnt = vq->vq_free_cnt;
1161 struct rte_mbuf *new_pkts[free_cnt];
1163 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1165 error = virtqueue_enqueue_recv_refill_packed(vq,
1166 new_pkts, free_cnt);
1167 if (unlikely(error)) {
1168 for (i = 0; i < free_cnt; i++)
1169 rte_pktmbuf_free(new_pkts[i]);
1171 nb_enqueued += free_cnt;
1173 struct rte_eth_dev *dev =
1174 &rte_eth_devices[rxvq->port_id];
1175 dev->data->rx_mbuf_alloc_failed += free_cnt;
1179 if (likely(nb_enqueued)) {
1180 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1181 virtqueue_notify(vq);
1182 PMD_RX_LOG(DEBUG, "Notified");
1191 virtio_recv_pkts_inorder(void *rx_queue,
1192 struct rte_mbuf **rx_pkts,
1195 struct virtnet_rx *rxvq = rx_queue;
1196 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1197 struct virtio_hw *hw = vq->hw;
1198 struct rte_mbuf *rxm;
1199 struct rte_mbuf *prev = NULL;
1200 uint16_t nb_used, num, nb_rx;
1201 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1202 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1204 uint32_t nb_enqueued;
1211 if (unlikely(hw->started == 0))
1214 nb_used = virtqueue_nused(vq);
1215 nb_used = RTE_MIN(nb_used, nb_pkts);
1216 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
1218 PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1223 hdr_size = hw->vtnet_hdr_size;
1225 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used);
1227 for (i = 0; i < num; i++) {
1228 struct virtio_net_hdr_mrg_rxbuf *header;
1230 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1231 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1235 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1236 PMD_RX_LOG(ERR, "Packet drop");
1238 virtio_discard_rxbuf_inorder(vq, rxm);
1239 rxvq->stats.errors++;
1243 header = (struct virtio_net_hdr_mrg_rxbuf *)
1244 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1247 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1248 seg_num = header->num_buffers;
1255 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1256 rxm->nb_segs = seg_num;
1259 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1260 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1262 rxm->port = rxvq->port_id;
1264 rx_pkts[nb_rx] = rxm;
1267 if (vq->hw->has_rx_offload &&
1268 virtio_rx_offload(rxm, &header->hdr) < 0) {
1269 virtio_discard_rxbuf_inorder(vq, rxm);
1270 rxvq->stats.errors++;
1275 rte_vlan_strip(rx_pkts[nb_rx]);
1277 seg_res = seg_num - 1;
1279 /* Merge remaining segments */
1280 while (seg_res != 0 && i < (num - 1)) {
1284 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1285 rxm->pkt_len = (uint32_t)(len[i]);
1286 rxm->data_len = (uint16_t)(len[i]);
1288 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1296 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1301 /* Last packet still need merge segments */
1302 while (seg_res != 0) {
1303 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1304 VIRTIO_MBUF_BURST_SZ);
1306 if (likely(virtqueue_nused(vq) >= rcv_cnt)) {
1307 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len,
1309 uint16_t extra_idx = 0;
1312 while (extra_idx < rcv_cnt) {
1313 rxm = rcv_pkts[extra_idx];
1315 RTE_PKTMBUF_HEADROOM - hdr_size;
1316 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1317 rxm->data_len = (uint16_t)(len[extra_idx]);
1320 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1326 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1331 "No enough segments for packet.");
1332 rte_pktmbuf_free(rx_pkts[nb_rx]);
1333 rxvq->stats.errors++;
1338 rxvq->stats.packets += nb_rx;
1340 /* Allocate new mbuf for the used descriptor */
1342 if (likely(!virtqueue_full(vq))) {
1343 /* free_cnt may include mrg descs */
1344 uint16_t free_cnt = vq->vq_free_cnt;
1345 struct rte_mbuf *new_pkts[free_cnt];
1347 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1348 error = virtqueue_enqueue_refill_inorder(vq, new_pkts,
1350 if (unlikely(error)) {
1351 for (i = 0; i < free_cnt; i++)
1352 rte_pktmbuf_free(new_pkts[i]);
1354 nb_enqueued += free_cnt;
1356 struct rte_eth_dev *dev =
1357 &rte_eth_devices[rxvq->port_id];
1358 dev->data->rx_mbuf_alloc_failed += free_cnt;
1362 if (likely(nb_enqueued)) {
1363 vq_update_avail_idx(vq);
1365 if (unlikely(virtqueue_kick_prepare(vq))) {
1366 virtqueue_notify(vq);
1367 PMD_RX_LOG(DEBUG, "Notified");
1375 virtio_recv_mergeable_pkts(void *rx_queue,
1376 struct rte_mbuf **rx_pkts,
1379 struct virtnet_rx *rxvq = rx_queue;
1380 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1381 struct virtio_hw *hw = vq->hw;
1382 struct rte_mbuf *rxm;
1383 struct rte_mbuf *prev = NULL;
1384 uint16_t nb_used, num, nb_rx = 0;
1385 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1386 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1388 uint32_t nb_enqueued = 0;
1389 uint32_t seg_num = 0;
1390 uint32_t seg_res = 0;
1391 uint32_t hdr_size = hw->vtnet_hdr_size;
1394 if (unlikely(hw->started == 0))
1397 nb_used = virtqueue_nused(vq);
1399 PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1401 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1402 if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1403 num = VIRTIO_MBUF_BURST_SZ;
1404 if (likely(num > DESC_PER_CACHELINE))
1405 num = num - ((vq->vq_used_cons_idx + num) %
1406 DESC_PER_CACHELINE);
1409 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1411 for (i = 0; i < num; i++) {
1412 struct virtio_net_hdr_mrg_rxbuf *header;
1414 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1415 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1419 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1420 PMD_RX_LOG(ERR, "Packet drop");
1422 virtio_discard_rxbuf(vq, rxm);
1423 rxvq->stats.errors++;
1427 header = (struct virtio_net_hdr_mrg_rxbuf *)
1428 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1430 seg_num = header->num_buffers;
1434 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1435 rxm->nb_segs = seg_num;
1438 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1439 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1441 rxm->port = rxvq->port_id;
1443 rx_pkts[nb_rx] = rxm;
1446 if (hw->has_rx_offload &&
1447 virtio_rx_offload(rxm, &header->hdr) < 0) {
1448 virtio_discard_rxbuf(vq, rxm);
1449 rxvq->stats.errors++;
1454 rte_vlan_strip(rx_pkts[nb_rx]);
1456 seg_res = seg_num - 1;
1458 /* Merge remaining segments */
1459 while (seg_res != 0 && i < (num - 1)) {
1463 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1464 rxm->pkt_len = (uint32_t)(len[i]);
1465 rxm->data_len = (uint16_t)(len[i]);
1467 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1475 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1480 /* Last packet still need merge segments */
1481 while (seg_res != 0) {
1482 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1483 VIRTIO_MBUF_BURST_SZ);
1485 if (likely(virtqueue_nused(vq) >= rcv_cnt)) {
1486 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len,
1488 uint16_t extra_idx = 0;
1491 while (extra_idx < rcv_cnt) {
1492 rxm = rcv_pkts[extra_idx];
1494 RTE_PKTMBUF_HEADROOM - hdr_size;
1495 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1496 rxm->data_len = (uint16_t)(len[extra_idx]);
1499 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1505 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1510 "No enough segments for packet.");
1511 rte_pktmbuf_free(rx_pkts[nb_rx]);
1512 rxvq->stats.errors++;
1517 rxvq->stats.packets += nb_rx;
1519 /* Allocate new mbuf for the used descriptor */
1520 if (likely(!virtqueue_full(vq))) {
1521 /* free_cnt may include mrg descs */
1522 uint16_t free_cnt = vq->vq_free_cnt;
1523 struct rte_mbuf *new_pkts[free_cnt];
1525 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1526 error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1528 if (unlikely(error)) {
1529 for (i = 0; i < free_cnt; i++)
1530 rte_pktmbuf_free(new_pkts[i]);
1532 nb_enqueued += free_cnt;
1534 struct rte_eth_dev *dev =
1535 &rte_eth_devices[rxvq->port_id];
1536 dev->data->rx_mbuf_alloc_failed += free_cnt;
1540 if (likely(nb_enqueued)) {
1541 vq_update_avail_idx(vq);
1543 if (unlikely(virtqueue_kick_prepare(vq))) {
1544 virtqueue_notify(vq);
1545 PMD_RX_LOG(DEBUG, "Notified");
1553 virtio_recv_mergeable_pkts_packed(void *rx_queue,
1554 struct rte_mbuf **rx_pkts,
1557 struct virtnet_rx *rxvq = rx_queue;
1558 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1559 struct virtio_hw *hw = vq->hw;
1560 struct rte_mbuf *rxm;
1561 struct rte_mbuf *prev = NULL;
1562 uint16_t num, nb_rx = 0;
1563 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1564 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1565 uint32_t nb_enqueued = 0;
1566 uint32_t seg_num = 0;
1567 uint32_t seg_res = 0;
1568 uint32_t hdr_size = hw->vtnet_hdr_size;
1572 if (unlikely(hw->started == 0))
1577 if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1578 num = VIRTIO_MBUF_BURST_SZ;
1579 if (likely(num > DESC_PER_CACHELINE))
1580 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1582 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1584 for (i = 0; i < num; i++) {
1585 struct virtio_net_hdr_mrg_rxbuf *header;
1587 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1588 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1592 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1593 PMD_RX_LOG(ERR, "Packet drop");
1595 virtio_discard_rxbuf(vq, rxm);
1596 rxvq->stats.errors++;
1600 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)
1601 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size);
1602 seg_num = header->num_buffers;
1607 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1608 rxm->nb_segs = seg_num;
1611 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1612 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1614 rxm->port = rxvq->port_id;
1615 rx_pkts[nb_rx] = rxm;
1618 if (hw->has_rx_offload &&
1619 virtio_rx_offload(rxm, &header->hdr) < 0) {
1620 virtio_discard_rxbuf(vq, rxm);
1621 rxvq->stats.errors++;
1626 rte_vlan_strip(rx_pkts[nb_rx]);
1628 seg_res = seg_num - 1;
1630 /* Merge remaining segments */
1631 while (seg_res != 0 && i < (num - 1)) {
1635 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1636 rxm->pkt_len = (uint32_t)(len[i]);
1637 rxm->data_len = (uint16_t)(len[i]);
1639 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1647 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1652 /* Last packet still need merge segments */
1653 while (seg_res != 0) {
1654 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1655 VIRTIO_MBUF_BURST_SZ);
1656 uint16_t extra_idx = 0;
1658 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts,
1660 if (unlikely(rcv_cnt == 0)) {
1661 PMD_RX_LOG(ERR, "No enough segments for packet.");
1662 rte_pktmbuf_free(rx_pkts[nb_rx]);
1663 rxvq->stats.errors++;
1667 while (extra_idx < rcv_cnt) {
1668 rxm = rcv_pkts[extra_idx];
1670 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1671 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1672 rxm->data_len = (uint16_t)(len[extra_idx]);
1676 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1681 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1686 rxvq->stats.packets += nb_rx;
1688 /* Allocate new mbuf for the used descriptor */
1689 if (likely(!virtqueue_full(vq))) {
1690 /* free_cnt may include mrg descs */
1691 uint16_t free_cnt = vq->vq_free_cnt;
1692 struct rte_mbuf *new_pkts[free_cnt];
1694 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1695 error = virtqueue_enqueue_recv_refill_packed(vq,
1696 new_pkts, free_cnt);
1697 if (unlikely(error)) {
1698 for (i = 0; i < free_cnt; i++)
1699 rte_pktmbuf_free(new_pkts[i]);
1701 nb_enqueued += free_cnt;
1703 struct rte_eth_dev *dev =
1704 &rte_eth_devices[rxvq->port_id];
1705 dev->data->rx_mbuf_alloc_failed += free_cnt;
1709 if (likely(nb_enqueued)) {
1710 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1711 virtqueue_notify(vq);
1712 PMD_RX_LOG(DEBUG, "Notified");
1720 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
1726 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1727 struct rte_mbuf *m = tx_pkts[nb_tx];
1729 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1730 error = rte_validate_tx_offload(m);
1731 if (unlikely(error)) {
1737 /* Do VLAN tag insertion */
1738 if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) {
1739 error = rte_vlan_insert(&m);
1740 /* rte_vlan_insert() may change pointer
1741 * even in the case of failure
1745 if (unlikely(error)) {
1751 error = rte_net_intel_cksum_prepare(m);
1752 if (unlikely(error)) {
1757 if (m->ol_flags & PKT_TX_TCP_SEG)
1758 virtio_tso_fix_cksum(m);
1765 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts,
1768 struct virtnet_tx *txvq = tx_queue;
1769 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1770 struct virtio_hw *hw = vq->hw;
1771 uint16_t hdr_size = hw->vtnet_hdr_size;
1773 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER);
1775 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1778 if (unlikely(nb_pkts < 1))
1781 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1783 if (nb_pkts > vq->vq_free_cnt)
1784 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt,
1787 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1788 struct rte_mbuf *txm = tx_pkts[nb_tx];
1789 int can_push = 0, use_indirect = 0, slots, need;
1791 /* optimize ring usage */
1792 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1793 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1794 rte_mbuf_refcnt_read(txm) == 1 &&
1795 RTE_MBUF_DIRECT(txm) &&
1796 txm->nb_segs == 1 &&
1797 rte_pktmbuf_headroom(txm) >= hdr_size &&
1798 rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1799 __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
1801 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
1802 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
1804 /* How many main ring entries are needed to this Tx?
1806 * any_layout => number of segments
1807 * default => number of segments + 1
1809 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
1810 need = slots - vq->vq_free_cnt;
1812 /* Positive value indicates it need free vring descriptors */
1813 if (unlikely(need > 0)) {
1814 virtio_xmit_cleanup_packed(vq, need, in_order);
1815 need = slots - vq->vq_free_cnt;
1816 if (unlikely(need > 0)) {
1818 "No free tx descriptors to transmit");
1823 /* Enqueue Packet buffers */
1825 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order);
1827 virtqueue_enqueue_xmit_packed(txvq, txm, slots,
1831 virtio_update_packet_stats(&txvq->stats, txm);
1834 txvq->stats.packets += nb_tx;
1836 if (likely(nb_tx)) {
1837 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1838 virtqueue_notify(vq);
1839 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
1847 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1849 struct virtnet_tx *txvq = tx_queue;
1850 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1851 struct virtio_hw *hw = vq->hw;
1852 uint16_t hdr_size = hw->vtnet_hdr_size;
1853 uint16_t nb_used, nb_tx = 0;
1855 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1858 if (unlikely(nb_pkts < 1))
1861 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1863 nb_used = virtqueue_nused(vq);
1865 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
1866 virtio_xmit_cleanup(vq, nb_used);
1868 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1869 struct rte_mbuf *txm = tx_pkts[nb_tx];
1870 int can_push = 0, use_indirect = 0, slots, need;
1872 /* optimize ring usage */
1873 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1874 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1875 rte_mbuf_refcnt_read(txm) == 1 &&
1876 RTE_MBUF_DIRECT(txm) &&
1877 txm->nb_segs == 1 &&
1878 rte_pktmbuf_headroom(txm) >= hdr_size &&
1879 rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1880 __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
1882 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
1883 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
1886 /* How many main ring entries are needed to this Tx?
1887 * any_layout => number of segments
1889 * default => number of segments + 1
1891 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
1892 need = slots - vq->vq_free_cnt;
1894 /* Positive value indicates it need free vring descriptors */
1895 if (unlikely(need > 0)) {
1896 nb_used = virtqueue_nused(vq);
1898 need = RTE_MIN(need, (int)nb_used);
1900 virtio_xmit_cleanup(vq, need);
1901 need = slots - vq->vq_free_cnt;
1902 if (unlikely(need > 0)) {
1904 "No free tx descriptors to transmit");
1909 /* Enqueue Packet buffers */
1910 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect,
1913 virtio_update_packet_stats(&txvq->stats, txm);
1916 txvq->stats.packets += nb_tx;
1918 if (likely(nb_tx)) {
1919 vq_update_avail_idx(vq);
1921 if (unlikely(virtqueue_kick_prepare(vq))) {
1922 virtqueue_notify(vq);
1923 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
1930 static __rte_always_inline int
1931 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need)
1933 uint16_t nb_used, nb_clean, nb_descs;
1935 nb_descs = vq->vq_free_cnt + need;
1936 nb_used = virtqueue_nused(vq);
1937 nb_clean = RTE_MIN(need, (int)nb_used);
1939 virtio_xmit_cleanup_inorder(vq, nb_clean);
1941 return nb_descs - vq->vq_free_cnt;
1945 virtio_xmit_pkts_inorder(void *tx_queue,
1946 struct rte_mbuf **tx_pkts,
1949 struct virtnet_tx *txvq = tx_queue;
1950 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1951 struct virtio_hw *hw = vq->hw;
1952 uint16_t hdr_size = hw->vtnet_hdr_size;
1953 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0;
1954 struct rte_mbuf *inorder_pkts[nb_pkts];
1957 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1960 if (unlikely(nb_pkts < 1))
1964 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1965 nb_used = virtqueue_nused(vq);
1967 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
1968 virtio_xmit_cleanup_inorder(vq, nb_used);
1970 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1971 struct rte_mbuf *txm = tx_pkts[nb_tx];
1974 /* optimize ring usage */
1975 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1976 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1977 rte_mbuf_refcnt_read(txm) == 1 &&
1978 RTE_MBUF_DIRECT(txm) &&
1979 txm->nb_segs == 1 &&
1980 rte_pktmbuf_headroom(txm) >= hdr_size &&
1981 rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1982 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) {
1983 inorder_pkts[nb_inorder_pkts] = txm;
1989 if (nb_inorder_pkts) {
1990 need = nb_inorder_pkts - vq->vq_free_cnt;
1991 if (unlikely(need > 0)) {
1992 need = virtio_xmit_try_cleanup_inorder(vq,
1994 if (unlikely(need > 0)) {
1996 "No free tx descriptors to "
2001 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2003 nb_inorder_pkts = 0;
2006 slots = txm->nb_segs + 1;
2007 need = slots - vq->vq_free_cnt;
2008 if (unlikely(need > 0)) {
2009 need = virtio_xmit_try_cleanup_inorder(vq, slots);
2011 if (unlikely(need > 0)) {
2013 "No free tx descriptors to transmit");
2017 /* Enqueue Packet buffers */
2018 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1);
2020 virtio_update_packet_stats(&txvq->stats, txm);
2023 /* Transmit all inorder packets */
2024 if (nb_inorder_pkts) {
2025 need = nb_inorder_pkts - vq->vq_free_cnt;
2026 if (unlikely(need > 0)) {
2027 need = virtio_xmit_try_cleanup_inorder(vq,
2029 if (unlikely(need > 0)) {
2031 "No free tx descriptors to transmit");
2032 nb_inorder_pkts = vq->vq_free_cnt;
2037 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2041 txvq->stats.packets += nb_tx;
2043 if (likely(nb_tx)) {
2044 vq_update_avail_idx(vq);
2046 if (unlikely(virtqueue_kick_prepare(vq))) {
2047 virtqueue_notify(vq);
2048 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
2058 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
2059 struct rte_mbuf **rx_pkts __rte_unused,
2060 uint16_t nb_pkts __rte_unused)
2066 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
2067 struct rte_mbuf **tx_pkts __rte_unused,
2068 uint16_t nb_pkts __rte_unused)