1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
10 #include <rte_atomic.h>
11 #include <rte_memory.h>
12 #include <rte_mempool.h>
16 #include "virtio_ring.h"
17 #include "virtio_logs.h"
18 #include "virtio_rxtx.h"
22 #define DEFAULT_TX_FREE_THRESH 32
23 #define DEFAULT_RX_FREE_THRESH 32
25 #define VIRTIO_MBUF_BURST_SZ 64
27 * Per virtio_ring.h in Linux.
28 * For virtio_pci on SMP, we don't need to order with respect to MMIO
29 * accesses through relaxed memory I/O windows, so thread_fence is
32 * For using virtio to talk to real devices (eg. vDPA) we do need real
36 virtio_mb(uint8_t weak_barriers)
39 rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
45 virtio_rmb(uint8_t weak_barriers)
48 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
54 virtio_wmb(uint8_t weak_barriers)
57 rte_atomic_thread_fence(__ATOMIC_RELEASE);
62 static inline uint16_t
63 virtqueue_fetch_flags_packed(struct vring_packed_desc *dp,
64 uint8_t weak_barriers)
69 /* x86 prefers to using rte_io_rmb over __atomic_load_n as it reports
70 * a better perf(~1.5%), which comes from the saved branch by the compiler.
71 * The if and else branch are identical on the platforms except Arm.
74 flags = __atomic_load_n(&dp->flags, __ATOMIC_ACQUIRE);
88 virtqueue_store_flags_packed(struct vring_packed_desc *dp,
89 uint16_t flags, uint8_t weak_barriers)
92 /* x86 prefers to using rte_io_wmb over __atomic_store_n as it reports
93 * a better perf(~1.5%), which comes from the saved branch by the compiler.
94 * The if and else branch are identical on the platforms except Arm.
97 __atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE);
108 #ifdef RTE_PMD_PACKET_PREFETCH
109 #define rte_packet_prefetch(p) rte_prefetch1(p)
111 #define rte_packet_prefetch(p) do {} while(0)
114 #define VIRTQUEUE_MAX_NAME_SZ 32
116 #define VTNET_SQ_RQ_QUEUE_IDX 0
117 #define VTNET_SQ_TQ_QUEUE_IDX 1
118 #define VTNET_SQ_CQ_QUEUE_IDX 2
120 enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 };
122 * The maximum virtqueue size is 2^15. Use that value as the end of
123 * descriptor chain terminator since it will never be a valid index
124 * in the descriptor table. This is used to verify we are correctly
125 * handling vq_free_cnt.
127 #define VQ_RING_DESC_CHAIN_END 32768
130 * Control the RX mode, ie. promiscuous, allmulti, etc...
131 * All commands require an "out" sg entry containing a 1 byte
132 * state value, zero = disable, non-zero = enable. Commands
133 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
134 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
136 #define VIRTIO_NET_CTRL_RX 0
137 #define VIRTIO_NET_CTRL_RX_PROMISC 0
138 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1
139 #define VIRTIO_NET_CTRL_RX_ALLUNI 2
140 #define VIRTIO_NET_CTRL_RX_NOMULTI 3
141 #define VIRTIO_NET_CTRL_RX_NOUNI 4
142 #define VIRTIO_NET_CTRL_RX_NOBCAST 5
147 * The MAC filter table is managed by the hypervisor, the guest should
148 * assume the size is infinite. Filtering should be considered
149 * non-perfect, ie. based on hypervisor resources, the guest may
150 * received packets from sources not specified in the filter list.
152 * In addition to the class/cmd header, the TABLE_SET command requires
153 * two out scatterlists. Each contains a 4 byte count of entries followed
154 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The
155 * first sg list contains unicast addresses, the second is for multicast.
156 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
159 * The ADDR_SET command requests one out scatterlist, it contains a
160 * 6 bytes MAC address. This functionality is present if the
161 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
163 struct virtio_net_ctrl_mac {
165 uint8_t macs[][RTE_ETHER_ADDR_LEN];
168 #define VIRTIO_NET_CTRL_MAC 1
169 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
170 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
173 * Control VLAN filtering
175 * The VLAN filter table is controlled via a simple ADD/DEL interface.
176 * VLAN IDs not added may be filtered by the hypervisor. Del is the
177 * opposite of add. Both commands expect an out entry containing a 2
178 * byte VLAN ID. VLAN filtering is available with the
179 * VIRTIO_NET_F_CTRL_VLAN feature bit.
181 #define VIRTIO_NET_CTRL_VLAN 2
182 #define VIRTIO_NET_CTRL_VLAN_ADD 0
183 #define VIRTIO_NET_CTRL_VLAN_DEL 1
186 * Control link announce acknowledgement
188 * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that
189 * driver has recevied the notification; device would clear the
190 * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives
193 #define VIRTIO_NET_CTRL_ANNOUNCE 3
194 #define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0
196 struct virtio_net_ctrl_hdr {
201 typedef uint8_t virtio_net_ctrl_ack;
203 #define VIRTIO_NET_OK 0
204 #define VIRTIO_NET_ERR 1
206 #define VIRTIO_MAX_CTRL_DATA 2048
208 struct virtio_pmd_ctrl {
209 struct virtio_net_ctrl_hdr hdr;
210 virtio_net_ctrl_ack status;
211 uint8_t data[VIRTIO_MAX_CTRL_DATA];
214 struct vq_desc_extra {
221 struct virtio_hw *hw; /**< virtio_hw structure pointer. */
224 /**< vring keeping desc, used and avail */
229 /**< vring keeping descs and events */
230 struct vring_packed ring;
231 bool used_wrap_counter;
232 uint16_t cached_flags; /**< cached flags for descs */
233 uint16_t event_flags_shadow;
237 uint16_t vq_used_cons_idx; /**< last consumed descriptor */
238 uint16_t vq_nentries; /**< vring desc numbers */
239 uint16_t vq_free_cnt; /**< num of desc available */
240 uint16_t vq_avail_idx; /**< sync until needed */
241 uint16_t vq_free_thresh; /**< free threshold */
243 void *vq_ring_virt_mem; /**< linear address of vring*/
244 unsigned int vq_ring_size;
247 struct virtnet_rx rxq;
248 struct virtnet_tx txq;
249 struct virtnet_ctl cq;
252 rte_iova_t vq_ring_mem; /**< physical address of vring,
253 * or virtual address for virtio_user. */
256 * Head of the free chain in the descriptor table. If
257 * there are no free descriptors, this will be set to
258 * VQ_RING_DESC_CHAIN_END.
260 uint16_t vq_desc_head_idx;
261 uint16_t vq_desc_tail_idx;
262 uint16_t vq_queue_index;
263 uint16_t offset; /**< relative offset to obtain addr in mbuf */
264 uint16_t *notify_addr;
265 struct rte_mbuf **sw_ring; /**< RX software ring. */
266 struct vq_desc_extra vq_descx[0];
269 /* If multiqueue is provided by host, then we suppport it. */
270 #define VIRTIO_NET_CTRL_MQ 4
271 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
272 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
273 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
276 * This is the first element of the scatter-gather list. If you don't
277 * specify GSO or CSUM features, you can simply ignore the header.
279 struct virtio_net_hdr {
280 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/
281 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */
283 #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */
284 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */
285 #define VIRTIO_NET_HDR_GSO_UDP 3 /**< GSO frame, IPv4 UDP (UFO) */
286 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /**< GSO frame, IPv6 TCP */
287 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /**< TCP has ECN set */
289 uint16_t hdr_len; /**< Ethernet + IP + tcp/udp hdrs */
290 uint16_t gso_size; /**< Bytes to append to hdr_len per frame */
291 uint16_t csum_start; /**< Position to start checksumming from */
292 uint16_t csum_offset; /**< Offset after that to place checksum */
296 * This is the version of the header to use when the MRG_RXBUF
297 * feature has been negotiated.
299 struct virtio_net_hdr_mrg_rxbuf {
300 struct virtio_net_hdr hdr;
301 uint16_t num_buffers; /**< Number of merged rx buffers */
304 /* Region reserved to allow for transmit header and indirect ring */
305 #define VIRTIO_MAX_TX_INDIRECT 8
306 struct virtio_tx_region {
307 struct virtio_net_hdr_mrg_rxbuf tx_hdr;
309 struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT];
310 struct vring_packed_desc
311 tx_packed_indir[VIRTIO_MAX_TX_INDIRECT];
316 desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq)
318 uint16_t used, avail, flags;
320 flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers);
321 used = !!(flags & VRING_PACKED_DESC_F_USED);
322 avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
324 return avail == used && used == vq->vq_packed.used_wrap_counter;
328 vring_desc_init_packed(struct virtqueue *vq, int n)
331 for (i = 0; i < n - 1; i++) {
332 vq->vq_packed.ring.desc[i].id = i;
333 vq->vq_descx[i].next = i + 1;
335 vq->vq_packed.ring.desc[i].id = i;
336 vq->vq_descx[i].next = VQ_RING_DESC_CHAIN_END;
339 /* Chain all the descriptors in the ring with an END */
341 vring_desc_init_split(struct vring_desc *dp, uint16_t n)
345 for (i = 0; i < n - 1; i++)
346 dp[i].next = (uint16_t)(i + 1);
347 dp[i].next = VQ_RING_DESC_CHAIN_END;
351 vring_desc_init_indirect_packed(struct vring_packed_desc *dp, int n)
354 for (i = 0; i < n; i++) {
355 dp[i].id = (uint16_t)i;
356 dp[i].flags = VRING_DESC_F_WRITE;
361 * Tell the backend not to interrupt us. Implementation for packed virtqueues.
364 virtqueue_disable_intr_packed(struct virtqueue *vq)
366 if (vq->vq_packed.event_flags_shadow != RING_EVENT_FLAGS_DISABLE) {
367 vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_DISABLE;
368 vq->vq_packed.ring.driver->desc_event_flags =
369 vq->vq_packed.event_flags_shadow;
374 * Tell the backend not to interrupt us. Implementation for split virtqueues.
377 virtqueue_disable_intr_split(struct virtqueue *vq)
379 vq->vq_split.ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
383 * Tell the backend not to interrupt us.
386 virtqueue_disable_intr(struct virtqueue *vq)
388 if (virtio_with_packed_queue(vq->hw))
389 virtqueue_disable_intr_packed(vq);
391 virtqueue_disable_intr_split(vq);
395 * Tell the backend to interrupt. Implementation for packed virtqueues.
398 virtqueue_enable_intr_packed(struct virtqueue *vq)
400 if (vq->vq_packed.event_flags_shadow == RING_EVENT_FLAGS_DISABLE) {
401 vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_ENABLE;
402 vq->vq_packed.ring.driver->desc_event_flags =
403 vq->vq_packed.event_flags_shadow;
408 * Tell the backend to interrupt. Implementation for split virtqueues.
411 virtqueue_enable_intr_split(struct virtqueue *vq)
413 vq->vq_split.ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT);
417 * Tell the backend to interrupt us.
420 virtqueue_enable_intr(struct virtqueue *vq)
422 if (virtio_with_packed_queue(vq->hw))
423 virtqueue_enable_intr_packed(vq);
425 virtqueue_enable_intr_split(vq);
429 * Dump virtqueue internal structures, for debug purpose only.
431 void virtqueue_dump(struct virtqueue *vq);
433 * Get all mbufs to be freed.
435 struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq);
437 /* Flush the elements in the used ring. */
438 void virtqueue_rxvq_flush(struct virtqueue *vq);
440 int virtqueue_rxvq_reset_packed(struct virtqueue *vq);
442 int virtqueue_txvq_reset_packed(struct virtqueue *vq);
445 virtqueue_full(const struct virtqueue *vq)
447 return vq->vq_free_cnt == 0;
451 virtio_get_queue_type(struct virtio_hw *hw, uint16_t vq_idx)
453 if (vq_idx == hw->max_queue_pairs * 2)
455 else if (vq_idx % 2 == 0)
461 /* virtqueue_nused has load-acquire or rte_io_rmb insed */
462 static inline uint16_t
463 virtqueue_nused(const struct virtqueue *vq)
467 if (vq->hw->weak_barriers) {
469 * x86 prefers to using rte_smp_rmb over __atomic_load_n as it
470 * reports a slightly better perf, which comes from the saved
471 * branch by the compiler.
472 * The if and else branches are identical with the smp and io
473 * barriers both defined as compiler barriers on x86.
475 #ifdef RTE_ARCH_X86_64
476 idx = vq->vq_split.ring.used->idx;
479 idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx,
483 idx = vq->vq_split.ring.used->idx;
486 return idx - vq->vq_used_cons_idx;
489 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx);
490 void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t used_idx);
491 void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
495 vq_update_avail_idx(struct virtqueue *vq)
497 if (vq->hw->weak_barriers) {
498 /* x86 prefers to using rte_smp_wmb over __atomic_store_n as
499 * it reports a slightly better perf, which comes from the
500 * saved branch by the compiler.
501 * The if and else branches are identical with the smp and
502 * io barriers both defined as compiler barriers on x86.
504 #ifdef RTE_ARCH_X86_64
506 vq->vq_split.ring.avail->idx = vq->vq_avail_idx;
508 __atomic_store_n(&vq->vq_split.ring.avail->idx,
509 vq->vq_avail_idx, __ATOMIC_RELEASE);
513 vq->vq_split.ring.avail->idx = vq->vq_avail_idx;
518 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx)
522 * Place the head of the descriptor chain into the next slot and make
523 * it usable to the host. The chain is made available now rather than
524 * deferring to virtqueue_notify() in the hopes that if the host is
525 * currently running on another CPU, we can keep it processing the new
528 avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1));
529 if (unlikely(vq->vq_split.ring.avail->ring[avail_idx] != desc_idx))
530 vq->vq_split.ring.avail->ring[avail_idx] = desc_idx;
535 virtqueue_kick_prepare(struct virtqueue *vq)
538 * Ensure updated avail->idx is visible to vhost before reading
541 virtio_mb(vq->hw->weak_barriers);
542 return !(vq->vq_split.ring.used->flags & VRING_USED_F_NO_NOTIFY);
546 virtqueue_kick_prepare_packed(struct virtqueue *vq)
551 * Ensure updated data is visible to vhost before reading the flags.
553 virtio_mb(vq->hw->weak_barriers);
554 flags = vq->vq_packed.ring.device->desc_event_flags;
556 return flags != RING_EVENT_FLAGS_DISABLE;
560 * virtqueue_kick_prepare*() or the virtio_wmb() should be called
561 * before this function to be sure that all the data is visible to vhost.
564 virtqueue_notify(struct virtqueue *vq)
566 VIRTIO_OPS(vq->hw)->notify_queue(vq->hw, vq);
569 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
570 #define VIRTQUEUE_DUMP(vq) do { \
571 uint16_t used_idx, nused; \
572 used_idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx, \
574 nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \
575 if (virtio_with_packed_queue((vq)->hw)) { \
576 PMD_INIT_LOG(DEBUG, \
577 "VQ: - size=%d; free=%d; used_cons_idx=%d; avail_idx=%d;" \
578 " cached_flags=0x%x; used_wrap_counter=%d", \
579 (vq)->vq_nentries, (vq)->vq_free_cnt, (vq)->vq_used_cons_idx, \
580 (vq)->vq_avail_idx, (vq)->vq_packed.cached_flags, \
581 (vq)->vq_packed.used_wrap_counter); \
584 PMD_INIT_LOG(DEBUG, \
585 "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \
586 " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \
587 " avail.flags=0x%x; used.flags=0x%x", \
588 (vq)->vq_nentries, (vq)->vq_free_cnt, nused, (vq)->vq_desc_head_idx, \
589 (vq)->vq_split.ring.avail->idx, (vq)->vq_used_cons_idx, \
590 __atomic_load_n(&(vq)->vq_split.ring.used->idx, __ATOMIC_RELAXED), \
591 (vq)->vq_split.ring.avail->flags, (vq)->vq_split.ring.used->flags); \
594 #define VIRTQUEUE_DUMP(vq) do { } while (0)
597 /* avoid write operation when necessary, to lessen cache issues */
598 #define ASSIGN_UNLESS_EQUAL(var, val) do { \
599 typeof(var) *const var_ = &(var); \
600 typeof(val) const val_ = (val); \
605 #define virtqueue_clear_net_hdr(hdr) do { \
606 typeof(hdr) hdr_ = (hdr); \
607 ASSIGN_UNLESS_EQUAL((hdr_)->csum_start, 0); \
608 ASSIGN_UNLESS_EQUAL((hdr_)->csum_offset, 0); \
609 ASSIGN_UNLESS_EQUAL((hdr_)->flags, 0); \
610 ASSIGN_UNLESS_EQUAL((hdr_)->gso_type, 0); \
611 ASSIGN_UNLESS_EQUAL((hdr_)->gso_size, 0); \
612 ASSIGN_UNLESS_EQUAL((hdr_)->hdr_len, 0); \
616 virtqueue_xmit_offload(struct virtio_net_hdr *hdr,
617 struct rte_mbuf *cookie,
621 if (cookie->ol_flags & PKT_TX_TCP_SEG)
622 cookie->ol_flags |= PKT_TX_TCP_CKSUM;
624 switch (cookie->ol_flags & PKT_TX_L4_MASK) {
625 case PKT_TX_UDP_CKSUM:
626 hdr->csum_start = cookie->l2_len + cookie->l3_len;
627 hdr->csum_offset = offsetof(struct rte_udp_hdr,
629 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
632 case PKT_TX_TCP_CKSUM:
633 hdr->csum_start = cookie->l2_len + cookie->l3_len;
634 hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum);
635 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
639 ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
640 ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
641 ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
645 /* TCP Segmentation Offload */
646 if (cookie->ol_flags & PKT_TX_TCP_SEG) {
647 hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ?
648 VIRTIO_NET_HDR_GSO_TCPV6 :
649 VIRTIO_NET_HDR_GSO_TCPV4;
650 hdr->gso_size = cookie->tso_segsz;
656 ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
657 ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
658 ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
664 virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
665 uint16_t needed, int use_indirect, int can_push,
668 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
669 struct vq_desc_extra *dxp;
670 struct virtqueue *vq = txvq->vq;
671 struct vring_packed_desc *start_dp, *head_dp;
672 uint16_t idx, id, head_idx, head_flags;
673 int16_t head_size = vq->hw->vtnet_hdr_size;
674 struct virtio_net_hdr *hdr;
676 bool prepend_header = false;
677 uint16_t seg_num = cookie->nb_segs;
679 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
681 dxp = &vq->vq_descx[id];
682 dxp->ndescs = needed;
683 dxp->cookie = cookie;
685 head_idx = vq->vq_avail_idx;
688 start_dp = vq->vq_packed.ring.desc;
690 head_dp = &vq->vq_packed.ring.desc[idx];
691 head_flags = cookie->next ? VRING_DESC_F_NEXT : 0;
692 head_flags |= vq->vq_packed.cached_flags;
695 /* prepend cannot fail, checked by caller */
696 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
698 prepend_header = true;
700 /* if offload disabled, it is not zeroed below, do it now */
701 if (!vq->hw->has_tx_offload)
702 virtqueue_clear_net_hdr(hdr);
703 } else if (use_indirect) {
704 /* setup tx ring slot to point to indirect
705 * descriptor list stored in reserved region.
707 * the first slot in indirect ring is already preset
708 * to point to the header in reserved region
710 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
711 RTE_PTR_DIFF(&txr[idx].tx_packed_indir, txr);
712 start_dp[idx].len = (seg_num + 1) *
713 sizeof(struct vring_packed_desc);
714 /* reset flags for indirect desc */
715 head_flags = VRING_DESC_F_INDIRECT;
716 head_flags |= vq->vq_packed.cached_flags;
717 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
719 /* loop below will fill in rest of the indirect elements */
720 start_dp = txr[idx].tx_packed_indir;
723 /* setup first tx ring slot to point to header
724 * stored in reserved region.
726 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
727 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
728 start_dp[idx].len = vq->hw->vtnet_hdr_size;
729 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
731 if (idx >= vq->vq_nentries) {
732 idx -= vq->vq_nentries;
733 vq->vq_packed.cached_flags ^=
734 VRING_PACKED_DESC_F_AVAIL_USED;
738 virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload);
743 start_dp[idx].addr = rte_mbuf_data_iova(cookie);
744 start_dp[idx].len = cookie->data_len;
745 if (prepend_header) {
746 start_dp[idx].addr -= head_size;
747 start_dp[idx].len += head_size;
748 prepend_header = false;
751 if (likely(idx != head_idx)) {
752 flags = cookie->next ? VRING_DESC_F_NEXT : 0;
753 flags |= vq->vq_packed.cached_flags;
754 start_dp[idx].flags = flags;
758 if (idx >= vq->vq_nentries) {
759 idx -= vq->vq_nentries;
760 vq->vq_packed.cached_flags ^=
761 VRING_PACKED_DESC_F_AVAIL_USED;
763 } while ((cookie = cookie->next) != NULL);
765 start_dp[prev].id = id;
769 if (++idx >= vq->vq_nentries) {
770 idx -= vq->vq_nentries;
771 vq->vq_packed.cached_flags ^=
772 VRING_PACKED_DESC_F_AVAIL_USED;
776 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
777 vq->vq_avail_idx = idx;
780 vq->vq_desc_head_idx = dxp->next;
781 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
782 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
785 virtqueue_store_flags_packed(head_dp, head_flags,
786 vq->hw->weak_barriers);
790 vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)
792 struct vq_desc_extra *dxp;
794 dxp = &vq->vq_descx[id];
795 vq->vq_free_cnt += dxp->ndescs;
797 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END)
798 vq->vq_desc_head_idx = id;
800 vq->vq_descx[vq->vq_desc_tail_idx].next = id;
802 vq->vq_desc_tail_idx = id;
803 dxp->next = VQ_RING_DESC_CHAIN_END;
807 virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
809 uint16_t used_idx, id, curr_id, free_cnt = 0;
810 uint16_t size = vq->vq_nentries;
811 struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
812 struct vq_desc_extra *dxp;
814 used_idx = vq->vq_used_cons_idx;
815 /* desc_is_used has a load-acquire or rte_io_rmb inside
816 * and wait for used desc in virtqueue.
818 while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
819 id = desc[used_idx].id;
822 dxp = &vq->vq_descx[used_idx];
823 used_idx += dxp->ndescs;
824 free_cnt += dxp->ndescs;
826 if (used_idx >= size) {
828 vq->vq_packed.used_wrap_counter ^= 1;
830 if (dxp->cookie != NULL) {
831 rte_pktmbuf_free(dxp->cookie);
834 } while (curr_id != id);
836 vq->vq_used_cons_idx = used_idx;
837 vq->vq_free_cnt += free_cnt;
841 virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
843 uint16_t used_idx, id;
844 uint16_t size = vq->vq_nentries;
845 struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
846 struct vq_desc_extra *dxp;
848 used_idx = vq->vq_used_cons_idx;
849 /* desc_is_used has a load-acquire or rte_io_rmb inside
850 * and wait for used desc in virtqueue.
852 while (num-- && desc_is_used(&desc[used_idx], vq)) {
853 id = desc[used_idx].id;
854 dxp = &vq->vq_descx[id];
855 vq->vq_used_cons_idx += dxp->ndescs;
856 if (vq->vq_used_cons_idx >= size) {
857 vq->vq_used_cons_idx -= size;
858 vq->vq_packed.used_wrap_counter ^= 1;
860 vq_ring_free_id_packed(vq, id);
861 if (dxp->cookie != NULL) {
862 rte_pktmbuf_free(dxp->cookie);
865 used_idx = vq->vq_used_cons_idx;
869 /* Cleanup from completed transmits. */
871 virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order)
874 virtio_xmit_cleanup_inorder_packed(vq, num);
876 virtio_xmit_cleanup_normal_packed(vq, num);
880 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
882 uint16_t i, used_idx, desc_idx;
883 for (i = 0; i < num; i++) {
884 struct vring_used_elem *uep;
885 struct vq_desc_extra *dxp;
887 used_idx = (uint16_t)(vq->vq_used_cons_idx &
888 (vq->vq_nentries - 1));
889 uep = &vq->vq_split.ring.used->ring[used_idx];
891 desc_idx = (uint16_t)uep->id;
892 dxp = &vq->vq_descx[desc_idx];
893 vq->vq_used_cons_idx++;
894 vq_ring_free_chain(vq, desc_idx);
896 if (dxp->cookie != NULL) {
897 rte_pktmbuf_free(dxp->cookie);
903 /* Cleanup from completed inorder transmits. */
904 static __rte_always_inline void
905 virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num)
907 uint16_t i, idx = vq->vq_used_cons_idx;
908 int16_t free_cnt = 0;
909 struct vq_desc_extra *dxp = NULL;
911 if (unlikely(num == 0))
914 for (i = 0; i < num; i++) {
915 dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)];
916 free_cnt += dxp->ndescs;
917 if (dxp->cookie != NULL) {
918 rte_pktmbuf_free(dxp->cookie);
923 vq->vq_free_cnt += free_cnt;
924 vq->vq_used_cons_idx = idx;
926 #endif /* _VIRTQUEUE_H_ */