1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016 Cavium, Inc
11 #include <rte_byteorder.h>
12 #include <rte_branch_prediction.h>
13 #include <rte_cycles.h>
14 #include <rte_ether.h>
15 #include <rte_ethdev.h>
16 #include <rte_errno.h>
17 #include <rte_memory.h>
18 #include <rte_mempool.h>
19 #include <rte_malloc.h>
21 #include <rte_prefetch.h>
22 #include <rte_string_fns.h>
25 #include "virtio_rxtx_simple.h"
27 #define RTE_VIRTIO_VPMD_RX_BURST 32
28 #define RTE_VIRTIO_DESC_PER_LOOP 8
29 #define RTE_VIRTIO_VPMD_RX_REARM_THRESH RTE_VIRTIO_VPMD_RX_BURST
31 /* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP)
33 * This routine is for non-mergeable RX, one desc for each guest buffer.
34 * This routine is based on the RX ring layout optimization. Each entry in the
35 * avail ring points to the desc with the same index in the desc ring and this
36 * will never be changed in the driver.
38 * - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet
41 virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
44 struct virtnet_rx *rxvq = rx_queue;
45 struct virtqueue *vq = rxvq->vq;
46 struct virtio_hw *hw = vq->hw;
49 struct vring_used_elem *rused;
50 struct rte_mbuf **sw_ring;
51 struct rte_mbuf **sw_ring_end;
52 uint16_t nb_pkts_received = 0;
54 uint8x16_t shuf_msk1 = {
55 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
56 4, 5, 0xFF, 0xFF, /* pkt len */
58 0xFF, 0xFF, /* vlan tci */
59 0xFF, 0xFF, 0xFF, 0xFF
62 uint8x16_t shuf_msk2 = {
63 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
64 12, 13, 0xFF, 0xFF, /* pkt len */
66 0xFF, 0xFF, /* vlan tci */
67 0xFF, 0xFF, 0xFF, 0xFF
70 /* Subtract the header length.
71 * In which case do we need the header length in used->len ?
73 uint16x8_t len_adjust = {
75 (uint16_t)vq->hw->vtnet_hdr_size, 0,
76 (uint16_t)vq->hw->vtnet_hdr_size,
81 if (unlikely(hw->started == 0))
82 return nb_pkts_received;
84 if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP))
87 nb_used = VIRTQUEUE_NUSED(vq);
91 if (unlikely(nb_used == 0))
94 nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP);
95 nb_used = RTE_MIN(nb_used, nb_pkts);
97 desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
98 rused = &vq->vq_ring.used->ring[desc_idx];
99 sw_ring = &vq->sw_ring[desc_idx];
100 sw_ring_end = &vq->sw_ring[vq->vq_nentries];
102 rte_prefetch_non_temporal(rused);
104 if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
105 virtio_rxq_rearm_vec(rxvq);
106 if (unlikely(virtqueue_kick_prepare(vq)))
107 virtqueue_notify(vq);
110 for (nb_pkts_received = 0;
111 nb_pkts_received < nb_used;) {
112 uint64x2_t desc[RTE_VIRTIO_DESC_PER_LOOP / 2];
113 uint64x2_t mbp[RTE_VIRTIO_DESC_PER_LOOP / 2];
114 uint64x2_t pkt_mb[RTE_VIRTIO_DESC_PER_LOOP];
116 mbp[0] = vld1q_u64((uint64_t *)(sw_ring + 0));
117 desc[0] = vld1q_u64((uint64_t *)(rused + 0));
118 vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0]);
120 mbp[1] = vld1q_u64((uint64_t *)(sw_ring + 2));
121 desc[1] = vld1q_u64((uint64_t *)(rused + 2));
122 vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1]);
124 mbp[2] = vld1q_u64((uint64_t *)(sw_ring + 4));
125 desc[2] = vld1q_u64((uint64_t *)(rused + 4));
126 vst1q_u64((uint64_t *)&rx_pkts[4], mbp[2]);
128 mbp[3] = vld1q_u64((uint64_t *)(sw_ring + 6));
129 desc[3] = vld1q_u64((uint64_t *)(rused + 6));
130 vst1q_u64((uint64_t *)&rx_pkts[6], mbp[3]);
132 pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
133 vreinterpretq_u8_u64(desc[0]), shuf_msk2));
134 pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
135 vreinterpretq_u8_u64(desc[0]), shuf_msk1));
136 pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
137 vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
138 pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
139 vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
140 vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1,
142 vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1,
145 pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
146 vreinterpretq_u8_u64(desc[1]), shuf_msk2));
147 pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
148 vreinterpretq_u8_u64(desc[1]), shuf_msk1));
149 pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
150 vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
151 pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
152 vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
153 vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1,
155 vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1,
158 pkt_mb[5] = vreinterpretq_u64_u8(vqtbl1q_u8(
159 vreinterpretq_u8_u64(desc[2]), shuf_msk2));
160 pkt_mb[4] = vreinterpretq_u64_u8(vqtbl1q_u8(
161 vreinterpretq_u8_u64(desc[2]), shuf_msk1));
162 pkt_mb[5] = vreinterpretq_u64_u16(vsubq_u16(
163 vreinterpretq_u16_u64(pkt_mb[5]), len_adjust));
164 pkt_mb[4] = vreinterpretq_u64_u16(vsubq_u16(
165 vreinterpretq_u16_u64(pkt_mb[4]), len_adjust));
166 vst1q_u64((void *)&rx_pkts[5]->rx_descriptor_fields1,
168 vst1q_u64((void *)&rx_pkts[4]->rx_descriptor_fields1,
171 pkt_mb[7] = vreinterpretq_u64_u8(vqtbl1q_u8(
172 vreinterpretq_u8_u64(desc[3]), shuf_msk2));
173 pkt_mb[6] = vreinterpretq_u64_u8(vqtbl1q_u8(
174 vreinterpretq_u8_u64(desc[3]), shuf_msk1));
175 pkt_mb[7] = vreinterpretq_u64_u16(vsubq_u16(
176 vreinterpretq_u16_u64(pkt_mb[7]), len_adjust));
177 pkt_mb[6] = vreinterpretq_u64_u16(vsubq_u16(
178 vreinterpretq_u16_u64(pkt_mb[6]), len_adjust));
179 vst1q_u64((void *)&rx_pkts[7]->rx_descriptor_fields1,
181 vst1q_u64((void *)&rx_pkts[6]->rx_descriptor_fields1,
184 if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) {
185 if (sw_ring + nb_used <= sw_ring_end)
186 nb_pkts_received += nb_used;
188 nb_pkts_received += sw_ring_end - sw_ring;
191 if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >=
193 nb_pkts_received += sw_ring_end - sw_ring;
196 nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP;
198 rx_pkts += RTE_VIRTIO_DESC_PER_LOOP;
199 sw_ring += RTE_VIRTIO_DESC_PER_LOOP;
200 rused += RTE_VIRTIO_DESC_PER_LOOP;
201 nb_used -= RTE_VIRTIO_DESC_PER_LOOP;
206 vq->vq_used_cons_idx += nb_pkts_received;
207 vq->vq_free_cnt += nb_pkts_received;
208 rxvq->stats.packets += nb_pkts_received;
209 return nb_pkts_received;