2411dbaee86681d5c803ce32e41b1ec6b1d09b4b
[dpdk.git] / drivers / net / virtio / virtio_rxtx.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <stdint.h>
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #include <errno.h>
39
40 #include <rte_cycles.h>
41 #include <rte_memory.h>
42 #include <rte_memzone.h>
43 #include <rte_branch_prediction.h>
44 #include <rte_mempool.h>
45 #include <rte_malloc.h>
46 #include <rte_mbuf.h>
47 #include <rte_ether.h>
48 #include <rte_ethdev.h>
49 #include <rte_prefetch.h>
50 #include <rte_string_fns.h>
51 #include <rte_errno.h>
52 #include <rte_byteorder.h>
53
54 #include "virtio_logs.h"
55 #include "virtio_ethdev.h"
56 #include "virtio_pci.h"
57 #include "virtqueue.h"
58 #include "virtio_rxtx.h"
59
60 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
61 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
62 #else
63 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
64 #endif
65
66
67 #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
68         ETH_TXQ_FLAGS_NOOFFLOADS)
69
70 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
71 static int use_simple_rxtx;
72 #endif
73
74 static void
75 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
76 {
77         struct vring_desc *dp, *dp_tail;
78         struct vq_desc_extra *dxp;
79         uint16_t desc_idx_last = desc_idx;
80
81         dp  = &vq->vq_ring.desc[desc_idx];
82         dxp = &vq->vq_descx[desc_idx];
83         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
84         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
85                 while (dp->flags & VRING_DESC_F_NEXT) {
86                         desc_idx_last = dp->next;
87                         dp = &vq->vq_ring.desc[dp->next];
88                 }
89         }
90         dxp->ndescs = 0;
91
92         /*
93          * We must append the existing free chain, if any, to the end of
94          * newly freed chain. If the virtqueue was completely used, then
95          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
96          */
97         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
98                 vq->vq_desc_head_idx = desc_idx;
99         } else {
100                 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
101                 dp_tail->next = desc_idx;
102         }
103
104         vq->vq_desc_tail_idx = desc_idx_last;
105         dp->next = VQ_RING_DESC_CHAIN_END;
106 }
107
108 static uint16_t
109 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
110                            uint32_t *len, uint16_t num)
111 {
112         struct vring_used_elem *uep;
113         struct rte_mbuf *cookie;
114         uint16_t used_idx, desc_idx;
115         uint16_t i;
116
117         /*  Caller does the check */
118         for (i = 0; i < num ; i++) {
119                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
120                 uep = &vq->vq_ring.used->ring[used_idx];
121                 desc_idx = (uint16_t) uep->id;
122                 len[i] = uep->len;
123                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
124
125                 if (unlikely(cookie == NULL)) {
126                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
127                                 vq->vq_used_cons_idx);
128                         break;
129                 }
130
131                 rte_prefetch0(cookie);
132                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
133                 rx_pkts[i]  = cookie;
134                 vq->vq_used_cons_idx++;
135                 vq_ring_free_chain(vq, desc_idx);
136                 vq->vq_descx[desc_idx].cookie = NULL;
137         }
138
139         return i;
140 }
141
142 #ifndef DEFAULT_TX_FREE_THRESH
143 #define DEFAULT_TX_FREE_THRESH 32
144 #endif
145
146 /* Cleanup from completed transmits. */
147 static void
148 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
149 {
150         uint16_t i, used_idx, desc_idx;
151         for (i = 0; i < num; i++) {
152                 struct vring_used_elem *uep;
153                 struct vq_desc_extra *dxp;
154
155                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
156                 uep = &vq->vq_ring.used->ring[used_idx];
157
158                 desc_idx = (uint16_t) uep->id;
159                 dxp = &vq->vq_descx[desc_idx];
160                 vq->vq_used_cons_idx++;
161                 vq_ring_free_chain(vq, desc_idx);
162
163                 if (dxp->cookie != NULL) {
164                         rte_pktmbuf_free(dxp->cookie);
165                         dxp->cookie = NULL;
166                 }
167         }
168 }
169
170
171 static inline int
172 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
173 {
174         struct vq_desc_extra *dxp;
175         struct virtio_hw *hw = vq->hw;
176         struct vring_desc *start_dp;
177         uint16_t needed = 1;
178         uint16_t head_idx, idx;
179
180         if (unlikely(vq->vq_free_cnt == 0))
181                 return -ENOSPC;
182         if (unlikely(vq->vq_free_cnt < needed))
183                 return -EMSGSIZE;
184
185         head_idx = vq->vq_desc_head_idx;
186         if (unlikely(head_idx >= vq->vq_nentries))
187                 return -EFAULT;
188
189         idx = head_idx;
190         dxp = &vq->vq_descx[idx];
191         dxp->cookie = (void *)cookie;
192         dxp->ndescs = needed;
193
194         start_dp = vq->vq_ring.desc;
195         start_dp[idx].addr =
196                 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM
197                 - hw->vtnet_hdr_size);
198         start_dp[idx].len =
199                 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
200         start_dp[idx].flags =  VRING_DESC_F_WRITE;
201         idx = start_dp[idx].next;
202         vq->vq_desc_head_idx = idx;
203         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
204                 vq->vq_desc_tail_idx = idx;
205         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
206         vq_update_avail_ring(vq, head_idx);
207
208         return 0;
209 }
210
211 static inline void
212 virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie,
213                        uint16_t needed, int use_indirect, int can_push)
214 {
215         struct vq_desc_extra *dxp;
216         struct vring_desc *start_dp;
217         uint16_t seg_num = cookie->nb_segs;
218         uint16_t head_idx, idx;
219         uint16_t head_size = txvq->hw->vtnet_hdr_size;
220         unsigned long offs;
221
222         head_idx = txvq->vq_desc_head_idx;
223         idx = head_idx;
224         dxp = &txvq->vq_descx[idx];
225         dxp->cookie = (void *)cookie;
226         dxp->ndescs = needed;
227
228         start_dp = txvq->vq_ring.desc;
229
230         if (can_push) {
231                 /* put on zero'd transmit header (no offloads) */
232                 void *hdr = rte_pktmbuf_prepend(cookie, head_size);
233
234                 memset(hdr, 0, head_size);
235         } else if (use_indirect) {
236                 /* setup tx ring slot to point to indirect
237                  * descriptor list stored in reserved region.
238                  *
239                  * the first slot in indirect ring is already preset
240                  * to point to the header in reserved region
241                  */
242                 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
243
244                 offs = idx * sizeof(struct virtio_tx_region)
245                         + offsetof(struct virtio_tx_region, tx_indir);
246
247                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem + offs;
248                 start_dp[idx].len   = (seg_num + 1) * sizeof(struct vring_desc);
249                 start_dp[idx].flags = VRING_DESC_F_INDIRECT;
250
251                 /* loop below will fill in rest of the indirect elements */
252                 start_dp = txr[idx].tx_indir;
253                 idx = 1;
254         } else {
255                 /* setup first tx ring slot to point to header
256                  * stored in reserved region.
257                  */
258                 offs = idx * sizeof(struct virtio_tx_region)
259                         + offsetof(struct virtio_tx_region, tx_hdr);
260
261                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem + offs;
262                 start_dp[idx].len   = txvq->hw->vtnet_hdr_size;
263                 start_dp[idx].flags = VRING_DESC_F_NEXT;
264                 idx = start_dp[idx].next;
265         }
266
267         do {
268                 start_dp[idx].addr  = rte_mbuf_data_dma_addr(cookie);
269                 start_dp[idx].len   = cookie->data_len;
270                 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0;
271                 idx = start_dp[idx].next;
272         } while ((cookie = cookie->next) != NULL);
273
274         if (use_indirect)
275                 idx = txvq->vq_ring.desc[head_idx].next;
276
277         txvq->vq_desc_head_idx = idx;
278         if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
279                 txvq->vq_desc_tail_idx = idx;
280         txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
281         vq_update_avail_ring(txvq, head_idx);
282 }
283
284 static void
285 virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
286 {
287         struct rte_mbuf *m;
288         int i, nbufs, error, size = vq->vq_nentries;
289         struct vring *vr = &vq->vq_ring;
290         uint8_t *ring_mem = vq->vq_ring_virt_mem;
291
292         PMD_INIT_FUNC_TRACE();
293
294         /*
295          * Reinitialise since virtio port might have been stopped and restarted
296          */
297         memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size);
298         vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
299         vq->vq_used_cons_idx = 0;
300         vq->vq_desc_head_idx = 0;
301         vq->vq_avail_idx = 0;
302         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
303         vq->vq_free_cnt = vq->vq_nentries;
304         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
305
306         vring_desc_init(vr->desc, size);
307
308         /*
309          * Disable device(host) interrupting guest
310          */
311         virtqueue_disable_intr(vq);
312
313         /* Only rx virtqueue needs mbufs to be allocated at initialization */
314         if (queue_type == VTNET_RQ) {
315                 if (vq->mpool == NULL)
316                         rte_exit(EXIT_FAILURE,
317                         "Cannot allocate initial mbufs for rx virtqueue");
318
319                 /* Allocate blank mbufs for the each rx descriptor */
320                 nbufs = 0;
321                 error = ENOSPC;
322
323 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
324                 if (use_simple_rxtx)
325                         for (i = 0; i < vq->vq_nentries; i++) {
326                                 vq->vq_ring.avail->ring[i] = i;
327                                 vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
328                         }
329 #endif
330                 memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf));
331                 for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++)
332                         vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf;
333
334                 while (!virtqueue_full(vq)) {
335                         m = rte_mbuf_raw_alloc(vq->mpool);
336                         if (m == NULL)
337                                 break;
338
339                         /******************************************
340                         *         Enqueue allocated buffers        *
341                         *******************************************/
342 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
343                         if (use_simple_rxtx)
344                                 error = virtqueue_enqueue_recv_refill_simple(vq, m);
345                         else
346 #endif
347                                 error = virtqueue_enqueue_recv_refill(vq, m);
348                         if (error) {
349                                 rte_pktmbuf_free(m);
350                                 break;
351                         }
352                         nbufs++;
353                 }
354
355                 vq_update_avail_idx(vq);
356
357                 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
358         } else if (queue_type == VTNET_TQ) {
359 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
360                 if (use_simple_rxtx) {
361                         int mid_idx  = vq->vq_nentries >> 1;
362                         for (i = 0; i < mid_idx; i++) {
363                                 vq->vq_ring.avail->ring[i] = i + mid_idx;
364                                 vq->vq_ring.desc[i + mid_idx].next = i;
365                                 vq->vq_ring.desc[i + mid_idx].addr =
366                                         vq->virtio_net_hdr_mem +
367                                         offsetof(struct virtio_tx_region, tx_hdr);
368                                 vq->vq_ring.desc[i + mid_idx].len =
369                                         vq->hw->vtnet_hdr_size;
370                                 vq->vq_ring.desc[i + mid_idx].flags =
371                                         VRING_DESC_F_NEXT;
372                                 vq->vq_ring.desc[i].flags = 0;
373                         }
374                         for (i = mid_idx; i < vq->vq_nentries; i++)
375                                 vq->vq_ring.avail->ring[i] = i;
376                 }
377 #endif
378         }
379 }
380
381 void
382 virtio_dev_cq_start(struct rte_eth_dev *dev)
383 {
384         struct virtio_hw *hw = dev->data->dev_private;
385
386         if (hw->cvq) {
387                 virtio_dev_vring_start(hw->cvq, VTNET_CQ);
388                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
389         }
390 }
391
392 void
393 virtio_dev_rxtx_start(struct rte_eth_dev *dev)
394 {
395         /*
396          * Start receive and transmit vrings
397          * -    Setup vring structure for all queues
398          * -    Initialize descriptor for the rx vring
399          * -    Allocate blank mbufs for the each rx descriptor
400          *
401          */
402         int i;
403
404         PMD_INIT_FUNC_TRACE();
405
406         /* Start rx vring. */
407         for (i = 0; i < dev->data->nb_rx_queues; i++) {
408                 virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ);
409                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
410         }
411
412         /* Start tx vring. */
413         for (i = 0; i < dev->data->nb_tx_queues; i++) {
414                 virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ);
415                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
416         }
417 }
418
419 int
420 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
421                         uint16_t queue_idx,
422                         uint16_t nb_desc,
423                         unsigned int socket_id,
424                         __rte_unused const struct rte_eth_rxconf *rx_conf,
425                         struct rte_mempool *mp)
426 {
427         uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
428         struct virtqueue *vq;
429         int ret;
430
431         PMD_INIT_FUNC_TRACE();
432         ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
433                         nb_desc, socket_id, &vq);
434         if (ret < 0) {
435                 PMD_INIT_LOG(ERR, "rvq initialization failed");
436                 return ret;
437         }
438
439         /* Create mempool for rx mbuf allocation */
440         vq->mpool = mp;
441
442         dev->data->rx_queues[queue_idx] = vq;
443
444 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
445         virtio_rxq_vec_setup(vq);
446 #endif
447
448         return 0;
449 }
450
451 void
452 virtio_dev_rx_queue_release(void *rxq)
453 {
454         virtio_dev_queue_release(rxq);
455 }
456
457 /*
458  * struct rte_eth_dev *dev: Used to update dev
459  * uint16_t nb_desc: Defaults to values read from config space
460  * unsigned int socket_id: Used to allocate memzone
461  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
462  * uint16_t queue_idx: Just used as an index in dev txq list
463  */
464 int
465 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
466                         uint16_t queue_idx,
467                         uint16_t nb_desc,
468                         unsigned int socket_id,
469                         const struct rte_eth_txconf *tx_conf)
470 {
471         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
472
473 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
474         struct virtio_hw *hw = dev->data->dev_private;
475 #endif
476         struct virtqueue *vq;
477         uint16_t tx_free_thresh;
478         int ret;
479
480         PMD_INIT_FUNC_TRACE();
481
482         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS)
483             != ETH_TXQ_FLAGS_NOXSUMS) {
484                 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n");
485                 return -EINVAL;
486         }
487
488 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
489         /* Use simple rx/tx func if single segment and no offloads */
490         if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS &&
491              !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
492                 PMD_INIT_LOG(INFO, "Using simple rx/tx path");
493                 dev->tx_pkt_burst = virtio_xmit_pkts_simple;
494                 dev->rx_pkt_burst = virtio_recv_pkts_vec;
495                 use_simple_rxtx = 1;
496         }
497 #endif
498
499         ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
500                         nb_desc, socket_id, &vq);
501         if (ret < 0) {
502                 PMD_INIT_LOG(ERR, "rvq initialization failed");
503                 return ret;
504         }
505
506         tx_free_thresh = tx_conf->tx_free_thresh;
507         if (tx_free_thresh == 0)
508                 tx_free_thresh =
509                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
510
511         if (tx_free_thresh >= (vq->vq_nentries - 3)) {
512                 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the "
513                         "number of TX entries minus 3 (%u)."
514                         " (tx_free_thresh=%u port=%u queue=%u)\n",
515                         vq->vq_nentries - 3,
516                         tx_free_thresh, dev->data->port_id, queue_idx);
517                 return -EINVAL;
518         }
519
520         vq->vq_free_thresh = tx_free_thresh;
521
522         dev->data->tx_queues[queue_idx] = vq;
523         return 0;
524 }
525
526 void
527 virtio_dev_tx_queue_release(void *txq)
528 {
529         virtio_dev_queue_release(txq);
530 }
531
532 static void
533 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
534 {
535         int error;
536         /*
537          * Requeue the discarded mbuf. This should always be
538          * successful since it was just dequeued.
539          */
540         error = virtqueue_enqueue_recv_refill(vq, m);
541         if (unlikely(error)) {
542                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
543                 rte_pktmbuf_free(m);
544         }
545 }
546
547 static void
548 virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf)
549 {
550         uint32_t s = mbuf->pkt_len;
551         struct ether_addr *ea;
552
553         if (s == 64) {
554                 vq->size_bins[1]++;
555         } else if (s > 64 && s < 1024) {
556                 uint32_t bin;
557
558                 /* count zeros, and offset into correct bin */
559                 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
560                 vq->size_bins[bin]++;
561         } else {
562                 if (s < 64)
563                         vq->size_bins[0]++;
564                 else if (s < 1519)
565                         vq->size_bins[6]++;
566                 else if (s >= 1519)
567                         vq->size_bins[7]++;
568         }
569
570         ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
571         if (is_multicast_ether_addr(ea)) {
572                 if (is_broadcast_ether_addr(ea))
573                         vq->broadcast++;
574                 else
575                         vq->multicast++;
576         }
577 }
578
579 #define VIRTIO_MBUF_BURST_SZ 64
580 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
581 uint16_t
582 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
583 {
584         struct virtqueue *rxvq = rx_queue;
585         struct virtio_hw *hw;
586         struct rte_mbuf *rxm, *new_mbuf;
587         uint16_t nb_used, num, nb_rx;
588         uint32_t len[VIRTIO_MBUF_BURST_SZ];
589         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
590         int error;
591         uint32_t i, nb_enqueued;
592         uint32_t hdr_size;
593
594         nb_used = VIRTQUEUE_NUSED(rxvq);
595
596         virtio_rmb();
597
598         num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
599         num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
600         if (likely(num > DESC_PER_CACHELINE))
601                 num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
602
603         num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num);
604         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
605
606         hw = rxvq->hw;
607         nb_rx = 0;
608         nb_enqueued = 0;
609         hdr_size = hw->vtnet_hdr_size;
610
611         for (i = 0; i < num ; i++) {
612                 rxm = rcv_pkts[i];
613
614                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
615
616                 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
617                         PMD_RX_LOG(ERR, "Packet drop");
618                         nb_enqueued++;
619                         virtio_discard_rxbuf(rxvq, rxm);
620                         rxvq->errors++;
621                         continue;
622                 }
623
624                 rxm->port = rxvq->port_id;
625                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
626                 rxm->ol_flags = 0;
627                 rxm->vlan_tci = 0;
628
629                 rxm->nb_segs = 1;
630                 rxm->next = NULL;
631                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
632                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
633
634                 if (hw->vlan_strip)
635                         rte_vlan_strip(rxm);
636
637                 VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
638
639                 rx_pkts[nb_rx++] = rxm;
640
641                 rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
642                 virtio_update_packet_stats(rxvq, rxm);
643         }
644
645         rxvq->packets += nb_rx;
646
647         /* Allocate new mbuf for the used descriptor */
648         error = ENOSPC;
649         while (likely(!virtqueue_full(rxvq))) {
650                 new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
651                 if (unlikely(new_mbuf == NULL)) {
652                         struct rte_eth_dev *dev
653                                 = &rte_eth_devices[rxvq->port_id];
654                         dev->data->rx_mbuf_alloc_failed++;
655                         break;
656                 }
657                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
658                 if (unlikely(error)) {
659                         rte_pktmbuf_free(new_mbuf);
660                         break;
661                 }
662                 nb_enqueued++;
663         }
664
665         if (likely(nb_enqueued)) {
666                 vq_update_avail_idx(rxvq);
667
668                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
669                         virtqueue_notify(rxvq);
670                         PMD_RX_LOG(DEBUG, "Notified");
671                 }
672         }
673
674         return nb_rx;
675 }
676
677 uint16_t
678 virtio_recv_mergeable_pkts(void *rx_queue,
679                         struct rte_mbuf **rx_pkts,
680                         uint16_t nb_pkts)
681 {
682         struct virtqueue *rxvq = rx_queue;
683         struct virtio_hw *hw;
684         struct rte_mbuf *rxm, *new_mbuf;
685         uint16_t nb_used, num, nb_rx;
686         uint32_t len[VIRTIO_MBUF_BURST_SZ];
687         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
688         struct rte_mbuf *prev;
689         int error;
690         uint32_t i, nb_enqueued;
691         uint32_t seg_num;
692         uint16_t extra_idx;
693         uint32_t seg_res;
694         uint32_t hdr_size;
695
696         nb_used = VIRTQUEUE_NUSED(rxvq);
697
698         virtio_rmb();
699
700         PMD_RX_LOG(DEBUG, "used:%d", nb_used);
701
702         hw = rxvq->hw;
703         nb_rx = 0;
704         i = 0;
705         nb_enqueued = 0;
706         seg_num = 0;
707         extra_idx = 0;
708         seg_res = 0;
709         hdr_size = hw->vtnet_hdr_size;
710
711         while (i < nb_used) {
712                 struct virtio_net_hdr_mrg_rxbuf *header;
713
714                 if (nb_rx == nb_pkts)
715                         break;
716
717                 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1);
718                 if (num != 1)
719                         continue;
720
721                 i++;
722
723                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
724                 PMD_RX_LOG(DEBUG, "packet len:%d", len[0]);
725
726                 rxm = rcv_pkts[0];
727
728                 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
729                         PMD_RX_LOG(ERR, "Packet drop");
730                         nb_enqueued++;
731                         virtio_discard_rxbuf(rxvq, rxm);
732                         rxvq->errors++;
733                         continue;
734                 }
735
736                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
737                         RTE_PKTMBUF_HEADROOM - hdr_size);
738                 seg_num = header->num_buffers;
739
740                 if (seg_num == 0)
741                         seg_num = 1;
742
743                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
744                 rxm->nb_segs = seg_num;
745                 rxm->next = NULL;
746                 rxm->ol_flags = 0;
747                 rxm->vlan_tci = 0;
748                 rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
749                 rxm->data_len = (uint16_t)(len[0] - hdr_size);
750
751                 rxm->port = rxvq->port_id;
752                 rx_pkts[nb_rx] = rxm;
753                 prev = rxm;
754
755                 seg_res = seg_num - 1;
756
757                 while (seg_res != 0) {
758                         /*
759                          * Get extra segments for current uncompleted packet.
760                          */
761                         uint16_t  rcv_cnt =
762                                 RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
763                         if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) {
764                                 uint32_t rx_num =
765                                         virtqueue_dequeue_burst_rx(rxvq,
766                                         rcv_pkts, len, rcv_cnt);
767                                 i += rx_num;
768                                 rcv_cnt = rx_num;
769                         } else {
770                                 PMD_RX_LOG(ERR,
771                                            "No enough segments for packet.");
772                                 nb_enqueued++;
773                                 virtio_discard_rxbuf(rxvq, rxm);
774                                 rxvq->errors++;
775                                 break;
776                         }
777
778                         extra_idx = 0;
779
780                         while (extra_idx < rcv_cnt) {
781                                 rxm = rcv_pkts[extra_idx];
782
783                                 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
784                                 rxm->next = NULL;
785                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
786                                 rxm->data_len = (uint16_t)(len[extra_idx]);
787
788                                 if (prev)
789                                         prev->next = rxm;
790
791                                 prev = rxm;
792                                 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len;
793                                 extra_idx++;
794                         };
795                         seg_res -= rcv_cnt;
796                 }
797
798                 if (hw->vlan_strip)
799                         rte_vlan_strip(rx_pkts[nb_rx]);
800
801                 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
802                         rx_pkts[nb_rx]->data_len);
803
804                 rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
805                 virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]);
806                 nb_rx++;
807         }
808
809         rxvq->packets += nb_rx;
810
811         /* Allocate new mbuf for the used descriptor */
812         error = ENOSPC;
813         while (likely(!virtqueue_full(rxvq))) {
814                 new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
815                 if (unlikely(new_mbuf == NULL)) {
816                         struct rte_eth_dev *dev
817                                 = &rte_eth_devices[rxvq->port_id];
818                         dev->data->rx_mbuf_alloc_failed++;
819                         break;
820                 }
821                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
822                 if (unlikely(error)) {
823                         rte_pktmbuf_free(new_mbuf);
824                         break;
825                 }
826                 nb_enqueued++;
827         }
828
829         if (likely(nb_enqueued)) {
830                 vq_update_avail_idx(rxvq);
831
832                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
833                         virtqueue_notify(rxvq);
834                         PMD_RX_LOG(DEBUG, "Notified");
835                 }
836         }
837
838         return nb_rx;
839 }
840
841 uint16_t
842 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
843 {
844         struct virtqueue *txvq = tx_queue;
845         struct virtio_hw *hw = txvq->hw;
846         uint16_t hdr_size = hw->vtnet_hdr_size;
847         uint16_t nb_used, nb_tx;
848         int error;
849
850         if (unlikely(nb_pkts < 1))
851                 return nb_pkts;
852
853         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
854         nb_used = VIRTQUEUE_NUSED(txvq);
855
856         virtio_rmb();
857         if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh))
858                 virtio_xmit_cleanup(txvq, nb_used);
859
860         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
861                 struct rte_mbuf *txm = tx_pkts[nb_tx];
862                 int can_push = 0, use_indirect = 0, slots, need;
863
864                 /* Do VLAN tag insertion */
865                 if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
866                         error = rte_vlan_insert(&txm);
867                         if (unlikely(error)) {
868                                 rte_pktmbuf_free(txm);
869                                 continue;
870                         }
871                 }
872
873                 /* optimize ring usage */
874                 if (vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) &&
875                     rte_mbuf_refcnt_read(txm) == 1 &&
876                     RTE_MBUF_DIRECT(txm) &&
877                     txm->nb_segs == 1 &&
878                     rte_pktmbuf_headroom(txm) >= hdr_size &&
879                     rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
880                                    __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
881                         can_push = 1;
882                 else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
883                          txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
884                         use_indirect = 1;
885
886                 /* How many main ring entries are needed to this Tx?
887                  * any_layout => number of segments
888                  * indirect   => 1
889                  * default    => number of segments + 1
890                  */
891                 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
892                 need = slots - txvq->vq_free_cnt;
893
894                 /* Positive value indicates it need free vring descriptors */
895                 if (unlikely(need > 0)) {
896                         nb_used = VIRTQUEUE_NUSED(txvq);
897                         virtio_rmb();
898                         need = RTE_MIN(need, (int)nb_used);
899
900                         virtio_xmit_cleanup(txvq, need);
901                         need = slots - txvq->vq_free_cnt;
902                         if (unlikely(need > 0)) {
903                                 PMD_TX_LOG(ERR,
904                                            "No free tx descriptors to transmit");
905                                 break;
906                         }
907                 }
908
909                 /* Enqueue Packet buffers */
910                 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, can_push);
911
912                 txvq->bytes += txm->pkt_len;
913                 virtio_update_packet_stats(txvq, txm);
914         }
915
916         txvq->packets += nb_tx;
917
918         if (likely(nb_tx)) {
919                 vq_update_avail_idx(txvq);
920
921                 if (unlikely(virtqueue_kick_prepare(txvq))) {
922                         virtqueue_notify(txvq);
923                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
924                 }
925         }
926
927         return nb_tx;
928 }