vhost: fix dequeue offload support
[dpdk.git] / lib / librte_vhost / virtio_net.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <stdint.h>
35 #include <stdbool.h>
36 #include <linux/virtio_net.h>
37
38 #include <rte_mbuf.h>
39 #include <rte_memcpy.h>
40 #include <rte_ether.h>
41 #include <rte_ip.h>
42 #include <rte_vhost.h>
43 #include <rte_tcp.h>
44 #include <rte_udp.h>
45 #include <rte_sctp.h>
46 #include <rte_arp.h>
47
48 #include "iotlb.h"
49 #include "vhost.h"
50
51 #define MAX_PKT_BURST 32
52
53 #define MAX_BATCH_LEN 256
54
55 static bool
56 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
57 {
58         return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring;
59 }
60
61 static __rte_always_inline void
62 do_flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
63                           uint16_t to, uint16_t from, uint16_t size)
64 {
65         rte_memcpy(&vq->used->ring[to],
66                         &vq->shadow_used_ring[from],
67                         size * sizeof(struct vring_used_elem));
68         vhost_log_used_vring(dev, vq,
69                         offsetof(struct vring_used, ring[to]),
70                         size * sizeof(struct vring_used_elem));
71 }
72
73 static __rte_always_inline void
74 flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq)
75 {
76         uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
77
78         if (used_idx + vq->shadow_used_idx <= vq->size) {
79                 do_flush_shadow_used_ring(dev, vq, used_idx, 0,
80                                           vq->shadow_used_idx);
81         } else {
82                 uint16_t size;
83
84                 /* update used ring interval [used_idx, vq->size] */
85                 size = vq->size - used_idx;
86                 do_flush_shadow_used_ring(dev, vq, used_idx, 0, size);
87
88                 /* update the left half used ring interval [0, left_size] */
89                 do_flush_shadow_used_ring(dev, vq, 0, size,
90                                           vq->shadow_used_idx - size);
91         }
92         vq->last_used_idx += vq->shadow_used_idx;
93
94         rte_smp_wmb();
95
96         *(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx;
97         vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
98                 sizeof(vq->used->idx));
99 }
100
101 static __rte_always_inline void
102 update_shadow_used_ring(struct vhost_virtqueue *vq,
103                          uint16_t desc_idx, uint16_t len)
104 {
105         uint16_t i = vq->shadow_used_idx++;
106
107         vq->shadow_used_ring[i].id  = desc_idx;
108         vq->shadow_used_ring[i].len = len;
109 }
110
111 static inline void
112 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
113 {
114         struct batch_copy_elem *elem = vq->batch_copy_elems;
115         uint16_t count = vq->batch_copy_nb_elems;
116         int i;
117
118         for (i = 0; i < count; i++) {
119                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
120                 vhost_log_write(dev, elem[i].log_addr, elem[i].len);
121                 PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
122         }
123 }
124
125 static inline void
126 do_data_copy_dequeue(struct vhost_virtqueue *vq)
127 {
128         struct batch_copy_elem *elem = vq->batch_copy_elems;
129         uint16_t count = vq->batch_copy_nb_elems;
130         int i;
131
132         for (i = 0; i < count; i++)
133                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
134 }
135
136 /* avoid write operation when necessary, to lessen cache issues */
137 #define ASSIGN_UNLESS_EQUAL(var, val) do {      \
138         if ((var) != (val))                     \
139                 (var) = (val);                  \
140 } while (0)
141
142 static void
143 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
144 {
145         uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK;
146
147         if (m_buf->ol_flags & PKT_TX_TCP_SEG)
148                 csum_l4 |= PKT_TX_TCP_CKSUM;
149
150         if (csum_l4) {
151                 net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
152                 net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len;
153
154                 switch (csum_l4) {
155                 case PKT_TX_TCP_CKSUM:
156                         net_hdr->csum_offset = (offsetof(struct tcp_hdr,
157                                                 cksum));
158                         break;
159                 case PKT_TX_UDP_CKSUM:
160                         net_hdr->csum_offset = (offsetof(struct udp_hdr,
161                                                 dgram_cksum));
162                         break;
163                 case PKT_TX_SCTP_CKSUM:
164                         net_hdr->csum_offset = (offsetof(struct sctp_hdr,
165                                                 cksum));
166                         break;
167                 }
168         } else {
169                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0);
170                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0);
171                 ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0);
172         }
173
174         /* IP cksum verification cannot be bypassed, then calculate here */
175         if (m_buf->ol_flags & PKT_TX_IP_CKSUM) {
176                 struct ipv4_hdr *ipv4_hdr;
177
178                 ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct ipv4_hdr *,
179                                                    m_buf->l2_len);
180                 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
181         }
182
183         if (m_buf->ol_flags & PKT_TX_TCP_SEG) {
184                 if (m_buf->ol_flags & PKT_TX_IPV4)
185                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
186                 else
187                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
188                 net_hdr->gso_size = m_buf->tso_segsz;
189                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len
190                                         + m_buf->l4_len;
191         } else {
192                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0);
193                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0);
194                 ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0);
195         }
196 }
197
198 static __rte_always_inline int
199 copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
200                   struct vring_desc *descs, struct rte_mbuf *m,
201                   uint16_t desc_idx, uint32_t size)
202 {
203         uint32_t desc_avail, desc_offset;
204         uint32_t mbuf_avail, mbuf_offset;
205         uint32_t cpy_len;
206         struct vring_desc *desc;
207         uint64_t desc_addr;
208         /* A counter to avoid desc dead loop chain */
209         uint16_t nr_desc = 1;
210         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
211         uint16_t copy_nb = vq->batch_copy_nb_elems;
212         int error = 0;
213
214         desc = &descs[desc_idx];
215         desc_addr = vhost_iova_to_vva(dev, vq, desc->addr,
216                                         desc->len, VHOST_ACCESS_RW);
217         /*
218          * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
219          * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
220          * otherwise stores offset on the stack instead of in a register.
221          */
222         if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) {
223                 error = -1;
224                 goto out;
225         }
226
227         rte_prefetch0((void *)(uintptr_t)desc_addr);
228
229         virtio_enqueue_offload(m, (struct virtio_net_hdr *)(uintptr_t)desc_addr);
230         vhost_log_write(dev, desc->addr, dev->vhost_hlen);
231         PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
232
233         desc_offset = dev->vhost_hlen;
234         desc_avail  = desc->len - dev->vhost_hlen;
235
236         mbuf_avail  = rte_pktmbuf_data_len(m);
237         mbuf_offset = 0;
238         while (mbuf_avail != 0 || m->next != NULL) {
239                 /* done with current mbuf, fetch next */
240                 if (mbuf_avail == 0) {
241                         m = m->next;
242
243                         mbuf_offset = 0;
244                         mbuf_avail  = rte_pktmbuf_data_len(m);
245                 }
246
247                 /* done with current desc buf, fetch next */
248                 if (desc_avail == 0) {
249                         if ((desc->flags & VRING_DESC_F_NEXT) == 0) {
250                                 /* Room in vring buffer is not enough */
251                                 error = -1;
252                                 goto out;
253                         }
254                         if (unlikely(desc->next >= size || ++nr_desc > size)) {
255                                 error = -1;
256                                 goto out;
257                         }
258
259                         desc = &descs[desc->next];
260                         desc_addr = vhost_iova_to_vva(dev, vq, desc->addr,
261                                                         desc->len,
262                                                         VHOST_ACCESS_RW);
263                         if (unlikely(!desc_addr)) {
264                                 error = -1;
265                                 goto out;
266                         }
267
268                         desc_offset = 0;
269                         desc_avail  = desc->len;
270                 }
271
272                 cpy_len = RTE_MIN(desc_avail, mbuf_avail);
273                 if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) {
274                         rte_memcpy((void *)((uintptr_t)(desc_addr +
275                                                         desc_offset)),
276                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
277                                 cpy_len);
278                         vhost_log_write(dev, desc->addr + desc_offset, cpy_len);
279                         PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
280                                      cpy_len, 0);
281                 } else {
282                         batch_copy[copy_nb].dst =
283                                 (void *)((uintptr_t)(desc_addr + desc_offset));
284                         batch_copy[copy_nb].src =
285                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
286                         batch_copy[copy_nb].log_addr = desc->addr + desc_offset;
287                         batch_copy[copy_nb].len = cpy_len;
288                         copy_nb++;
289                 }
290
291                 mbuf_avail  -= cpy_len;
292                 mbuf_offset += cpy_len;
293                 desc_avail  -= cpy_len;
294                 desc_offset += cpy_len;
295         }
296
297 out:
298         vq->batch_copy_nb_elems = copy_nb;
299
300         return error;
301 }
302
303 /**
304  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
305  * be received from the physical port or from another virtio device. A packet
306  * count is returned to indicate the number of packets that are successfully
307  * added to the RX queue. This function works when the mbuf is scattered, but
308  * it doesn't support the mergeable feature.
309  */
310 static __rte_always_inline uint32_t
311 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
312               struct rte_mbuf **pkts, uint32_t count)
313 {
314         struct vhost_virtqueue *vq;
315         uint16_t avail_idx, free_entries, start_idx;
316         uint16_t desc_indexes[MAX_PKT_BURST];
317         struct vring_desc *descs;
318         uint16_t used_idx;
319         uint32_t i, sz;
320
321         LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
322         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
323                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
324                         dev->vid, __func__, queue_id);
325                 return 0;
326         }
327
328         vq = dev->virtqueue[queue_id];
329         if (unlikely(vq->enabled == 0))
330                 return 0;
331
332         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
333                 vhost_user_iotlb_rd_lock(vq);
334
335         if (unlikely(vq->access_ok == 0)) {
336                 if (unlikely(vring_translate(dev, vq) < 0)) {
337                         count = 0;
338                         goto out;
339                 }
340         }
341
342         avail_idx = *((volatile uint16_t *)&vq->avail->idx);
343         start_idx = vq->last_used_idx;
344         free_entries = avail_idx - start_idx;
345         count = RTE_MIN(count, free_entries);
346         count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST);
347         if (count == 0)
348                 goto out;
349
350         LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n",
351                 dev->vid, start_idx, start_idx + count);
352
353         vq->batch_copy_nb_elems = 0;
354
355         /* Retrieve all of the desc indexes first to avoid caching issues. */
356         rte_prefetch0(&vq->avail->ring[start_idx & (vq->size - 1)]);
357         for (i = 0; i < count; i++) {
358                 used_idx = (start_idx + i) & (vq->size - 1);
359                 desc_indexes[i] = vq->avail->ring[used_idx];
360                 vq->used->ring[used_idx].id = desc_indexes[i];
361                 vq->used->ring[used_idx].len = pkts[i]->pkt_len +
362                                                dev->vhost_hlen;
363                 vhost_log_used_vring(dev, vq,
364                         offsetof(struct vring_used, ring[used_idx]),
365                         sizeof(vq->used->ring[used_idx]));
366         }
367
368         rte_prefetch0(&vq->desc[desc_indexes[0]]);
369         for (i = 0; i < count; i++) {
370                 uint16_t desc_idx = desc_indexes[i];
371                 int err;
372
373                 if (vq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) {
374                         descs = (struct vring_desc *)(uintptr_t)
375                                 vhost_iova_to_vva(dev,
376                                                 vq, vq->desc[desc_idx].addr,
377                                                 vq->desc[desc_idx].len,
378                                                 VHOST_ACCESS_RO);
379                         if (unlikely(!descs)) {
380                                 count = i;
381                                 break;
382                         }
383
384                         desc_idx = 0;
385                         sz = vq->desc[desc_idx].len / sizeof(*descs);
386                 } else {
387                         descs = vq->desc;
388                         sz = vq->size;
389                 }
390
391                 err = copy_mbuf_to_desc(dev, vq, descs, pkts[i], desc_idx, sz);
392                 if (unlikely(err)) {
393                         count = i;
394                         break;
395                 }
396
397                 if (i + 1 < count)
398                         rte_prefetch0(&vq->desc[desc_indexes[i+1]]);
399         }
400
401         do_data_copy_enqueue(dev, vq);
402
403         rte_smp_wmb();
404
405         *(volatile uint16_t *)&vq->used->idx += count;
406         vq->last_used_idx += count;
407         vhost_log_used_vring(dev, vq,
408                 offsetof(struct vring_used, idx),
409                 sizeof(vq->used->idx));
410
411         /* flush used->idx update before we read avail->flags. */
412         rte_mb();
413
414         /* Kick the guest if necessary. */
415         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
416                         && (vq->callfd >= 0))
417                 eventfd_write(vq->callfd, (eventfd_t)1);
418 out:
419         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
420                 vhost_user_iotlb_rd_unlock(vq);
421
422         return count;
423 }
424
425 static __rte_always_inline int
426 fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
427                          uint32_t avail_idx, uint32_t *vec_idx,
428                          struct buf_vector *buf_vec, uint16_t *desc_chain_head,
429                          uint16_t *desc_chain_len)
430 {
431         uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
432         uint32_t vec_id = *vec_idx;
433         uint32_t len    = 0;
434         struct vring_desc *descs = vq->desc;
435
436         *desc_chain_head = idx;
437
438         if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
439                 descs = (struct vring_desc *)(uintptr_t)
440                         vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
441                                                 vq->desc[idx].len,
442                                                 VHOST_ACCESS_RO);
443                 if (unlikely(!descs))
444                         return -1;
445
446                 idx = 0;
447         }
448
449         while (1) {
450                 if (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size))
451                         return -1;
452
453                 len += descs[idx].len;
454                 buf_vec[vec_id].buf_addr = descs[idx].addr;
455                 buf_vec[vec_id].buf_len  = descs[idx].len;
456                 buf_vec[vec_id].desc_idx = idx;
457                 vec_id++;
458
459                 if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
460                         break;
461
462                 idx = descs[idx].next;
463         }
464
465         *desc_chain_len = len;
466         *vec_idx = vec_id;
467
468         return 0;
469 }
470
471 /*
472  * Returns -1 on fail, 0 on success
473  */
474 static inline int
475 reserve_avail_buf_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
476                                 uint32_t size, struct buf_vector *buf_vec,
477                                 uint16_t *num_buffers, uint16_t avail_head)
478 {
479         uint16_t cur_idx;
480         uint32_t vec_idx = 0;
481         uint16_t tries = 0;
482
483         uint16_t head_idx = 0;
484         uint16_t len = 0;
485
486         *num_buffers = 0;
487         cur_idx  = vq->last_avail_idx;
488
489         while (size > 0) {
490                 if (unlikely(cur_idx == avail_head))
491                         return -1;
492
493                 if (unlikely(fill_vec_buf(dev, vq, cur_idx, &vec_idx, buf_vec,
494                                                 &head_idx, &len) < 0))
495                         return -1;
496                 len = RTE_MIN(len, size);
497                 update_shadow_used_ring(vq, head_idx, len);
498                 size -= len;
499
500                 cur_idx++;
501                 tries++;
502                 *num_buffers += 1;
503
504                 /*
505                  * if we tried all available ring items, and still
506                  * can't get enough buf, it means something abnormal
507                  * happened.
508                  */
509                 if (unlikely(tries >= vq->size))
510                         return -1;
511         }
512
513         return 0;
514 }
515
516 static __rte_always_inline int
517 copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
518                             struct rte_mbuf *m, struct buf_vector *buf_vec,
519                             uint16_t num_buffers)
520 {
521         uint32_t vec_idx = 0;
522         uint64_t desc_addr;
523         uint32_t mbuf_offset, mbuf_avail;
524         uint32_t desc_offset, desc_avail;
525         uint32_t cpy_len;
526         uint64_t hdr_addr, hdr_phys_addr;
527         struct rte_mbuf *hdr_mbuf;
528         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
529         uint16_t copy_nb = vq->batch_copy_nb_elems;
530         int error = 0;
531
532         if (unlikely(m == NULL)) {
533                 error = -1;
534                 goto out;
535         }
536
537         desc_addr = vhost_iova_to_vva(dev, vq, buf_vec[vec_idx].buf_addr,
538                                                 buf_vec[vec_idx].buf_len,
539                                                 VHOST_ACCESS_RW);
540         if (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) {
541                 error = -1;
542                 goto out;
543         }
544
545         hdr_mbuf = m;
546         hdr_addr = desc_addr;
547         hdr_phys_addr = buf_vec[vec_idx].buf_addr;
548         rte_prefetch0((void *)(uintptr_t)hdr_addr);
549
550         LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
551                 dev->vid, num_buffers);
552
553         desc_avail  = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
554         desc_offset = dev->vhost_hlen;
555
556         mbuf_avail  = rte_pktmbuf_data_len(m);
557         mbuf_offset = 0;
558         while (mbuf_avail != 0 || m->next != NULL) {
559                 /* done with current desc buf, get the next one */
560                 if (desc_avail == 0) {
561                         vec_idx++;
562                         desc_addr =
563                                 vhost_iova_to_vva(dev, vq,
564                                         buf_vec[vec_idx].buf_addr,
565                                         buf_vec[vec_idx].buf_len,
566                                         VHOST_ACCESS_RW);
567                         if (unlikely(!desc_addr)) {
568                                 error = -1;
569                                 goto out;
570                         }
571
572                         /* Prefetch buffer address. */
573                         rte_prefetch0((void *)(uintptr_t)desc_addr);
574                         desc_offset = 0;
575                         desc_avail  = buf_vec[vec_idx].buf_len;
576                 }
577
578                 /* done with current mbuf, get the next one */
579                 if (mbuf_avail == 0) {
580                         m = m->next;
581
582                         mbuf_offset = 0;
583                         mbuf_avail  = rte_pktmbuf_data_len(m);
584                 }
585
586                 if (hdr_addr) {
587                         struct virtio_net_hdr_mrg_rxbuf *hdr;
588
589                         hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)
590                                 hdr_addr;
591                         virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
592                         ASSIGN_UNLESS_EQUAL(hdr->num_buffers, num_buffers);
593
594                         vhost_log_write(dev, hdr_phys_addr, dev->vhost_hlen);
595                         PRINT_PACKET(dev, (uintptr_t)hdr_addr,
596                                      dev->vhost_hlen, 0);
597
598                         hdr_addr = 0;
599                 }
600
601                 cpy_len = RTE_MIN(desc_avail, mbuf_avail);
602
603                 if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) {
604                         rte_memcpy((void *)((uintptr_t)(desc_addr +
605                                                         desc_offset)),
606                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
607                                 cpy_len);
608                         vhost_log_write(dev,
609                                 buf_vec[vec_idx].buf_addr + desc_offset,
610                                 cpy_len);
611                         PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
612                                 cpy_len, 0);
613                 } else {
614                         batch_copy[copy_nb].dst =
615                                 (void *)((uintptr_t)(desc_addr + desc_offset));
616                         batch_copy[copy_nb].src =
617                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
618                         batch_copy[copy_nb].log_addr =
619                                 buf_vec[vec_idx].buf_addr + desc_offset;
620                         batch_copy[copy_nb].len = cpy_len;
621                         copy_nb++;
622                 }
623
624                 mbuf_avail  -= cpy_len;
625                 mbuf_offset += cpy_len;
626                 desc_avail  -= cpy_len;
627                 desc_offset += cpy_len;
628         }
629
630 out:
631         vq->batch_copy_nb_elems = copy_nb;
632
633         return error;
634 }
635
636 static __rte_always_inline uint32_t
637 virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
638         struct rte_mbuf **pkts, uint32_t count)
639 {
640         struct vhost_virtqueue *vq;
641         uint32_t pkt_idx = 0;
642         uint16_t num_buffers;
643         struct buf_vector buf_vec[BUF_VECTOR_MAX];
644         uint16_t avail_head;
645
646         LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
647         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
648                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
649                         dev->vid, __func__, queue_id);
650                 return 0;
651         }
652
653         vq = dev->virtqueue[queue_id];
654         if (unlikely(vq->enabled == 0))
655                 return 0;
656
657         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
658                 vhost_user_iotlb_rd_lock(vq);
659
660         if (unlikely(vq->access_ok == 0))
661                 if (unlikely(vring_translate(dev, vq) < 0))
662                         goto out;
663
664         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
665         if (count == 0)
666                 goto out;
667
668         vq->batch_copy_nb_elems = 0;
669
670         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
671
672         vq->shadow_used_idx = 0;
673         avail_head = *((volatile uint16_t *)&vq->avail->idx);
674         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
675                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
676
677                 if (unlikely(reserve_avail_buf_mergeable(dev, vq,
678                                                 pkt_len, buf_vec, &num_buffers,
679                                                 avail_head) < 0)) {
680                         LOG_DEBUG(VHOST_DATA,
681                                 "(%d) failed to get enough desc from vring\n",
682                                 dev->vid);
683                         vq->shadow_used_idx -= num_buffers;
684                         break;
685                 }
686
687                 LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
688                         dev->vid, vq->last_avail_idx,
689                         vq->last_avail_idx + num_buffers);
690
691                 if (copy_mbuf_to_desc_mergeable(dev, vq, pkts[pkt_idx],
692                                                 buf_vec, num_buffers) < 0) {
693                         vq->shadow_used_idx -= num_buffers;
694                         break;
695                 }
696
697                 vq->last_avail_idx += num_buffers;
698         }
699
700         do_data_copy_enqueue(dev, vq);
701
702         if (likely(vq->shadow_used_idx)) {
703                 flush_shadow_used_ring(dev, vq);
704
705                 /* flush used->idx update before we read avail->flags. */
706                 rte_mb();
707
708                 /* Kick the guest if necessary. */
709                 if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
710                                 && (vq->callfd >= 0))
711                         eventfd_write(vq->callfd, (eventfd_t)1);
712         }
713
714 out:
715         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
716                 vhost_user_iotlb_rd_unlock(vq);
717
718         return pkt_idx;
719 }
720
721 uint16_t
722 rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
723         struct rte_mbuf **pkts, uint16_t count)
724 {
725         struct virtio_net *dev = get_device(vid);
726
727         if (!dev)
728                 return 0;
729
730         if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF))
731                 return virtio_dev_merge_rx(dev, queue_id, pkts, count);
732         else
733                 return virtio_dev_rx(dev, queue_id, pkts, count);
734 }
735
736 static inline bool
737 virtio_net_with_host_offload(struct virtio_net *dev)
738 {
739         if (dev->features &
740                         ((1ULL << VIRTIO_NET_F_CSUM) |
741                          (1ULL << VIRTIO_NET_F_HOST_ECN) |
742                          (1ULL << VIRTIO_NET_F_HOST_TSO4) |
743                          (1ULL << VIRTIO_NET_F_HOST_TSO6) |
744                          (1ULL << VIRTIO_NET_F_HOST_UFO)))
745                 return true;
746
747         return false;
748 }
749
750 static void
751 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
752 {
753         struct ipv4_hdr *ipv4_hdr;
754         struct ipv6_hdr *ipv6_hdr;
755         void *l3_hdr = NULL;
756         struct ether_hdr *eth_hdr;
757         uint16_t ethertype;
758
759         eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
760
761         m->l2_len = sizeof(struct ether_hdr);
762         ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
763
764         if (ethertype == ETHER_TYPE_VLAN) {
765                 struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
766
767                 m->l2_len += sizeof(struct vlan_hdr);
768                 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
769         }
770
771         l3_hdr = (char *)eth_hdr + m->l2_len;
772
773         switch (ethertype) {
774         case ETHER_TYPE_IPv4:
775                 ipv4_hdr = l3_hdr;
776                 *l4_proto = ipv4_hdr->next_proto_id;
777                 m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
778                 *l4_hdr = (char *)l3_hdr + m->l3_len;
779                 m->ol_flags |= PKT_TX_IPV4;
780                 break;
781         case ETHER_TYPE_IPv6:
782                 ipv6_hdr = l3_hdr;
783                 *l4_proto = ipv6_hdr->proto;
784                 m->l3_len = sizeof(struct ipv6_hdr);
785                 *l4_hdr = (char *)l3_hdr + m->l3_len;
786                 m->ol_flags |= PKT_TX_IPV6;
787                 break;
788         default:
789                 m->l3_len = 0;
790                 *l4_proto = 0;
791                 *l4_hdr = NULL;
792                 break;
793         }
794 }
795
796 static __rte_always_inline void
797 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
798 {
799         uint16_t l4_proto = 0;
800         void *l4_hdr = NULL;
801         struct tcp_hdr *tcp_hdr = NULL;
802
803         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
804                 return;
805
806         parse_ethernet(m, &l4_proto, &l4_hdr);
807         if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
808                 if (hdr->csum_start == (m->l2_len + m->l3_len)) {
809                         switch (hdr->csum_offset) {
810                         case (offsetof(struct tcp_hdr, cksum)):
811                                 if (l4_proto == IPPROTO_TCP)
812                                         m->ol_flags |= PKT_TX_TCP_CKSUM;
813                                 break;
814                         case (offsetof(struct udp_hdr, dgram_cksum)):
815                                 if (l4_proto == IPPROTO_UDP)
816                                         m->ol_flags |= PKT_TX_UDP_CKSUM;
817                                 break;
818                         case (offsetof(struct sctp_hdr, cksum)):
819                                 if (l4_proto == IPPROTO_SCTP)
820                                         m->ol_flags |= PKT_TX_SCTP_CKSUM;
821                                 break;
822                         default:
823                                 break;
824                         }
825                 }
826         }
827
828         if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
829                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
830                 case VIRTIO_NET_HDR_GSO_TCPV4:
831                 case VIRTIO_NET_HDR_GSO_TCPV6:
832                         tcp_hdr = l4_hdr;
833                         m->ol_flags |= PKT_TX_TCP_SEG;
834                         m->tso_segsz = hdr->gso_size;
835                         m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
836                         break;
837                 default:
838                         RTE_LOG(WARNING, VHOST_DATA,
839                                 "unsupported gso type %u.\n", hdr->gso_type);
840                         break;
841                 }
842         }
843 }
844
845 #define RARP_PKT_SIZE   64
846
847 static int
848 make_rarp_packet(struct rte_mbuf *rarp_mbuf, const struct ether_addr *mac)
849 {
850         struct ether_hdr *eth_hdr;
851         struct arp_hdr  *rarp;
852
853         if (rarp_mbuf->buf_len < 64) {
854                 RTE_LOG(WARNING, VHOST_DATA,
855                         "failed to make RARP; mbuf size too small %u (< %d)\n",
856                         rarp_mbuf->buf_len, RARP_PKT_SIZE);
857                 return -1;
858         }
859
860         /* Ethernet header. */
861         eth_hdr = rte_pktmbuf_mtod_offset(rarp_mbuf, struct ether_hdr *, 0);
862         memset(eth_hdr->d_addr.addr_bytes, 0xff, ETHER_ADDR_LEN);
863         ether_addr_copy(mac, &eth_hdr->s_addr);
864         eth_hdr->ether_type = htons(ETHER_TYPE_RARP);
865
866         /* RARP header. */
867         rarp = (struct arp_hdr *)(eth_hdr + 1);
868         rarp->arp_hrd = htons(ARP_HRD_ETHER);
869         rarp->arp_pro = htons(ETHER_TYPE_IPv4);
870         rarp->arp_hln = ETHER_ADDR_LEN;
871         rarp->arp_pln = 4;
872         rarp->arp_op  = htons(ARP_OP_REVREQUEST);
873
874         ether_addr_copy(mac, &rarp->arp_data.arp_sha);
875         ether_addr_copy(mac, &rarp->arp_data.arp_tha);
876         memset(&rarp->arp_data.arp_sip, 0x00, 4);
877         memset(&rarp->arp_data.arp_tip, 0x00, 4);
878
879         rarp_mbuf->pkt_len  = rarp_mbuf->data_len = RARP_PKT_SIZE;
880
881         return 0;
882 }
883
884 static __rte_always_inline void
885 put_zmbuf(struct zcopy_mbuf *zmbuf)
886 {
887         zmbuf->in_use = 0;
888 }
889
890 static __rte_always_inline int
891 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
892                   struct vring_desc *descs, uint16_t max_desc,
893                   struct rte_mbuf *m, uint16_t desc_idx,
894                   struct rte_mempool *mbuf_pool)
895 {
896         struct vring_desc *desc;
897         uint64_t desc_addr;
898         uint32_t desc_avail, desc_offset;
899         uint32_t mbuf_avail, mbuf_offset;
900         uint32_t cpy_len;
901         struct rte_mbuf *cur = m, *prev = m;
902         struct virtio_net_hdr *hdr = NULL;
903         /* A counter to avoid desc dead loop chain */
904         uint32_t nr_desc = 1;
905         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
906         uint16_t copy_nb = vq->batch_copy_nb_elems;
907         int error = 0;
908
909         desc = &descs[desc_idx];
910         if (unlikely((desc->len < dev->vhost_hlen)) ||
911                         (desc->flags & VRING_DESC_F_INDIRECT)) {
912                 error = -1;
913                 goto out;
914         }
915
916         desc_addr = vhost_iova_to_vva(dev,
917                                         vq, desc->addr,
918                                         desc->len,
919                                         VHOST_ACCESS_RO);
920         if (unlikely(!desc_addr)) {
921                 error = -1;
922                 goto out;
923         }
924
925         if (virtio_net_with_host_offload(dev)) {
926                 hdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr);
927                 rte_prefetch0(hdr);
928         }
929
930         /*
931          * A virtio driver normally uses at least 2 desc buffers
932          * for Tx: the first for storing the header, and others
933          * for storing the data.
934          */
935         if (likely((desc->len == dev->vhost_hlen) &&
936                    (desc->flags & VRING_DESC_F_NEXT) != 0)) {
937                 desc = &descs[desc->next];
938                 if (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) {
939                         error = -1;
940                         goto out;
941                 }
942
943                 desc_addr = vhost_iova_to_vva(dev,
944                                                         vq, desc->addr,
945                                                         desc->len,
946                                                         VHOST_ACCESS_RO);
947                 if (unlikely(!desc_addr)) {
948                         error = -1;
949                         goto out;
950                 }
951
952                 desc_offset = 0;
953                 desc_avail  = desc->len;
954                 nr_desc    += 1;
955         } else {
956                 desc_avail  = desc->len - dev->vhost_hlen;
957                 desc_offset = dev->vhost_hlen;
958         }
959
960         rte_prefetch0((void *)(uintptr_t)(desc_addr + desc_offset));
961
962         PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), desc_avail, 0);
963
964         mbuf_offset = 0;
965         mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
966         while (1) {
967                 uint64_t hpa;
968
969                 cpy_len = RTE_MIN(desc_avail, mbuf_avail);
970
971                 /*
972                  * A desc buf might across two host physical pages that are
973                  * not continuous. In such case (gpa_to_hpa returns 0), data
974                  * will be copied even though zero copy is enabled.
975                  */
976                 if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,
977                                         desc->addr + desc_offset, cpy_len)))) {
978                         cur->data_len = cpy_len;
979                         cur->data_off = 0;
980                         cur->buf_addr = (void *)(uintptr_t)desc_addr;
981                         cur->buf_physaddr = hpa;
982
983                         /*
984                          * In zero copy mode, one mbuf can only reference data
985                          * for one or partial of one desc buff.
986                          */
987                         mbuf_avail = cpy_len;
988                 } else {
989                         if (likely(cpy_len > MAX_BATCH_LEN ||
990                                    copy_nb >= vq->size ||
991                                    (hdr && cur == m))) {
992                                 rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
993                                                                    mbuf_offset),
994                                            (void *)((uintptr_t)(desc_addr +
995                                                                 desc_offset)),
996                                            cpy_len);
997                         } else {
998                                 batch_copy[copy_nb].dst =
999                                         rte_pktmbuf_mtod_offset(cur, void *,
1000                                                                 mbuf_offset);
1001                                 batch_copy[copy_nb].src =
1002                                         (void *)((uintptr_t)(desc_addr +
1003                                                              desc_offset));
1004                                 batch_copy[copy_nb].len = cpy_len;
1005                                 copy_nb++;
1006                         }
1007                 }
1008
1009                 mbuf_avail  -= cpy_len;
1010                 mbuf_offset += cpy_len;
1011                 desc_avail  -= cpy_len;
1012                 desc_offset += cpy_len;
1013
1014                 /* This desc reaches to its end, get the next one */
1015                 if (desc_avail == 0) {
1016                         if ((desc->flags & VRING_DESC_F_NEXT) == 0)
1017                                 break;
1018
1019                         if (unlikely(desc->next >= max_desc ||
1020                                      ++nr_desc > max_desc)) {
1021                                 error = -1;
1022                                 goto out;
1023                         }
1024                         desc = &descs[desc->next];
1025                         if (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) {
1026                                 error = -1;
1027                                 goto out;
1028                         }
1029
1030                         desc_addr = vhost_iova_to_vva(dev,
1031                                                         vq, desc->addr,
1032                                                         desc->len,
1033                                                         VHOST_ACCESS_RO);
1034                         if (unlikely(!desc_addr)) {
1035                                 error = -1;
1036                                 goto out;
1037                         }
1038
1039                         rte_prefetch0((void *)(uintptr_t)desc_addr);
1040
1041                         desc_offset = 0;
1042                         desc_avail  = desc->len;
1043
1044                         PRINT_PACKET(dev, (uintptr_t)desc_addr, desc->len, 0);
1045                 }
1046
1047                 /*
1048                  * This mbuf reaches to its end, get a new one
1049                  * to hold more data.
1050                  */
1051                 if (mbuf_avail == 0) {
1052                         cur = rte_pktmbuf_alloc(mbuf_pool);
1053                         if (unlikely(cur == NULL)) {
1054                                 RTE_LOG(ERR, VHOST_DATA, "Failed to "
1055                                         "allocate memory for mbuf.\n");
1056                                 error = -1;
1057                                 goto out;
1058                         }
1059                         if (unlikely(dev->dequeue_zero_copy))
1060                                 rte_mbuf_refcnt_update(cur, 1);
1061
1062                         prev->next = cur;
1063                         prev->data_len = mbuf_offset;
1064                         m->nb_segs += 1;
1065                         m->pkt_len += mbuf_offset;
1066                         prev = cur;
1067
1068                         mbuf_offset = 0;
1069                         mbuf_avail  = cur->buf_len - RTE_PKTMBUF_HEADROOM;
1070                 }
1071         }
1072
1073         prev->data_len = mbuf_offset;
1074         m->pkt_len    += mbuf_offset;
1075
1076         if (hdr)
1077                 vhost_dequeue_offload(hdr, m);
1078
1079 out:
1080         vq->batch_copy_nb_elems = copy_nb;
1081
1082         return error;
1083 }
1084
1085 static __rte_always_inline void
1086 update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
1087                  uint32_t used_idx, uint32_t desc_idx)
1088 {
1089         vq->used->ring[used_idx].id  = desc_idx;
1090         vq->used->ring[used_idx].len = 0;
1091         vhost_log_used_vring(dev, vq,
1092                         offsetof(struct vring_used, ring[used_idx]),
1093                         sizeof(vq->used->ring[used_idx]));
1094 }
1095
1096 static __rte_always_inline void
1097 update_used_idx(struct virtio_net *dev, struct vhost_virtqueue *vq,
1098                 uint32_t count)
1099 {
1100         if (unlikely(count == 0))
1101                 return;
1102
1103         rte_smp_wmb();
1104         rte_smp_rmb();
1105
1106         vq->used->idx += count;
1107         vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
1108                         sizeof(vq->used->idx));
1109
1110         /* Kick guest if required. */
1111         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
1112                         && (vq->callfd >= 0))
1113                 eventfd_write(vq->callfd, (eventfd_t)1);
1114 }
1115
1116 static __rte_always_inline struct zcopy_mbuf *
1117 get_zmbuf(struct vhost_virtqueue *vq)
1118 {
1119         uint16_t i;
1120         uint16_t last;
1121         int tries = 0;
1122
1123         /* search [last_zmbuf_idx, zmbuf_size) */
1124         i = vq->last_zmbuf_idx;
1125         last = vq->zmbuf_size;
1126
1127 again:
1128         for (; i < last; i++) {
1129                 if (vq->zmbufs[i].in_use == 0) {
1130                         vq->last_zmbuf_idx = i + 1;
1131                         vq->zmbufs[i].in_use = 1;
1132                         return &vq->zmbufs[i];
1133                 }
1134         }
1135
1136         tries++;
1137         if (tries == 1) {
1138                 /* search [0, last_zmbuf_idx) */
1139                 i = 0;
1140                 last = vq->last_zmbuf_idx;
1141                 goto again;
1142         }
1143
1144         return NULL;
1145 }
1146
1147 static __rte_always_inline bool
1148 mbuf_is_consumed(struct rte_mbuf *m)
1149 {
1150         while (m) {
1151                 if (rte_mbuf_refcnt_read(m) > 1)
1152                         return false;
1153                 m = m->next;
1154         }
1155
1156         return true;
1157 }
1158
1159 uint16_t
1160 rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
1161         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1162 {
1163         struct virtio_net *dev;
1164         struct rte_mbuf *rarp_mbuf = NULL;
1165         struct vhost_virtqueue *vq;
1166         uint32_t desc_indexes[MAX_PKT_BURST];
1167         uint32_t used_idx;
1168         uint32_t i = 0;
1169         uint16_t free_entries;
1170         uint16_t avail_idx;
1171
1172         dev = get_device(vid);
1173         if (!dev)
1174                 return 0;
1175
1176         if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
1177                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
1178                         dev->vid, __func__, queue_id);
1179                 return 0;
1180         }
1181
1182         vq = dev->virtqueue[queue_id];
1183         if (unlikely(vq->enabled == 0))
1184                 return 0;
1185
1186         vq->batch_copy_nb_elems = 0;
1187
1188         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1189                 vhost_user_iotlb_rd_lock(vq);
1190
1191         if (unlikely(vq->access_ok == 0))
1192                 if (unlikely(vring_translate(dev, vq) < 0))
1193                         goto out;
1194
1195         if (unlikely(dev->dequeue_zero_copy)) {
1196                 struct zcopy_mbuf *zmbuf, *next;
1197                 int nr_updated = 0;
1198
1199                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1200                      zmbuf != NULL; zmbuf = next) {
1201                         next = TAILQ_NEXT(zmbuf, next);
1202
1203                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1204                                 used_idx = vq->last_used_idx++ & (vq->size - 1);
1205                                 update_used_ring(dev, vq, used_idx,
1206                                                  zmbuf->desc_idx);
1207                                 nr_updated += 1;
1208
1209                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1210                                 rte_pktmbuf_free(zmbuf->mbuf);
1211                                 put_zmbuf(zmbuf);
1212                                 vq->nr_zmbuf -= 1;
1213                         }
1214                 }
1215
1216                 update_used_idx(dev, vq, nr_updated);
1217         }
1218
1219         /*
1220          * Construct a RARP broadcast packet, and inject it to the "pkts"
1221          * array, to looks like that guest actually send such packet.
1222          *
1223          * Check user_send_rarp() for more information.
1224          *
1225          * broadcast_rarp shares a cacheline in the virtio_net structure
1226          * with some fields that are accessed during enqueue and
1227          * rte_atomic16_cmpset() causes a write if using cmpxchg. This could
1228          * result in false sharing between enqueue and dequeue.
1229          *
1230          * Prevent unnecessary false sharing by reading broadcast_rarp first
1231          * and only performing cmpset if the read indicates it is likely to
1232          * be set.
1233          */
1234
1235         if (unlikely(rte_atomic16_read(&dev->broadcast_rarp) &&
1236                         rte_atomic16_cmpset((volatile uint16_t *)
1237                                 &dev->broadcast_rarp.cnt, 1, 0))) {
1238
1239                 rarp_mbuf = rte_pktmbuf_alloc(mbuf_pool);
1240                 if (rarp_mbuf == NULL) {
1241                         RTE_LOG(ERR, VHOST_DATA,
1242                                 "Failed to allocate memory for mbuf.\n");
1243                         return 0;
1244                 }
1245
1246                 if (make_rarp_packet(rarp_mbuf, &dev->mac)) {
1247                         rte_pktmbuf_free(rarp_mbuf);
1248                         rarp_mbuf = NULL;
1249                 } else {
1250                         count -= 1;
1251                 }
1252         }
1253
1254         free_entries = *((volatile uint16_t *)&vq->avail->idx) -
1255                         vq->last_avail_idx;
1256         if (free_entries == 0)
1257                 goto out;
1258
1259         LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1260
1261         /* Prefetch available and used ring */
1262         avail_idx = vq->last_avail_idx & (vq->size - 1);
1263         used_idx  = vq->last_used_idx  & (vq->size - 1);
1264         rte_prefetch0(&vq->avail->ring[avail_idx]);
1265         rte_prefetch0(&vq->used->ring[used_idx]);
1266
1267         count = RTE_MIN(count, MAX_PKT_BURST);
1268         count = RTE_MIN(count, free_entries);
1269         LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
1270                         dev->vid, count);
1271
1272         /* Retrieve all of the head indexes first to avoid caching issues. */
1273         for (i = 0; i < count; i++) {
1274                 avail_idx = (vq->last_avail_idx + i) & (vq->size - 1);
1275                 used_idx  = (vq->last_used_idx  + i) & (vq->size - 1);
1276                 desc_indexes[i] = vq->avail->ring[avail_idx];
1277
1278                 if (likely(dev->dequeue_zero_copy == 0))
1279                         update_used_ring(dev, vq, used_idx, desc_indexes[i]);
1280         }
1281
1282         /* Prefetch descriptor index. */
1283         rte_prefetch0(&vq->desc[desc_indexes[0]]);
1284         for (i = 0; i < count; i++) {
1285                 struct vring_desc *desc;
1286                 uint16_t sz, idx;
1287                 int err;
1288
1289                 if (likely(i + 1 < count))
1290                         rte_prefetch0(&vq->desc[desc_indexes[i + 1]]);
1291
1292                 if (vq->desc[desc_indexes[i]].flags & VRING_DESC_F_INDIRECT) {
1293                         desc = (struct vring_desc *)(uintptr_t)
1294                                 vhost_iova_to_vva(dev, vq,
1295                                                 vq->desc[desc_indexes[i]].addr,
1296                                                 sizeof(*desc),
1297                                                 VHOST_ACCESS_RO);
1298                         if (unlikely(!desc))
1299                                 break;
1300
1301                         rte_prefetch0(desc);
1302                         sz = vq->desc[desc_indexes[i]].len / sizeof(*desc);
1303                         idx = 0;
1304                 } else {
1305                         desc = vq->desc;
1306                         sz = vq->size;
1307                         idx = desc_indexes[i];
1308                 }
1309
1310                 pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
1311                 if (unlikely(pkts[i] == NULL)) {
1312                         RTE_LOG(ERR, VHOST_DATA,
1313                                 "Failed to allocate memory for mbuf.\n");
1314                         break;
1315                 }
1316
1317                 err = copy_desc_to_mbuf(dev, vq, desc, sz, pkts[i], idx,
1318                                         mbuf_pool);
1319                 if (unlikely(err)) {
1320                         rte_pktmbuf_free(pkts[i]);
1321                         break;
1322                 }
1323
1324                 if (unlikely(dev->dequeue_zero_copy)) {
1325                         struct zcopy_mbuf *zmbuf;
1326
1327                         zmbuf = get_zmbuf(vq);
1328                         if (!zmbuf) {
1329                                 rte_pktmbuf_free(pkts[i]);
1330                                 break;
1331                         }
1332                         zmbuf->mbuf = pkts[i];
1333                         zmbuf->desc_idx = desc_indexes[i];
1334
1335                         /*
1336                          * Pin lock the mbuf; we will check later to see
1337                          * whether the mbuf is freed (when we are the last
1338                          * user) or not. If that's the case, we then could
1339                          * update the used ring safely.
1340                          */
1341                         rte_mbuf_refcnt_update(pkts[i], 1);
1342
1343                         vq->nr_zmbuf += 1;
1344                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1345                 }
1346         }
1347         vq->last_avail_idx += i;
1348
1349         if (likely(dev->dequeue_zero_copy == 0)) {
1350                 do_data_copy_dequeue(vq);
1351                 vq->last_used_idx += i;
1352                 update_used_idx(dev, vq, i);
1353         }
1354
1355 out:
1356         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1357                 vhost_user_iotlb_rd_unlock(vq);
1358
1359         if (unlikely(rarp_mbuf != NULL)) {
1360                 /*
1361                  * Inject it to the head of "pkts" array, so that switch's mac
1362                  * learning table will get updated first.
1363                  */
1364                 memmove(&pkts[1], pkts, i * sizeof(struct rte_mbuf *));
1365                 pkts[0] = rarp_mbuf;
1366                 i += 1;
1367         }
1368
1369         return i;
1370 }