net/virtio: split Rx/Tx queue
[dpdk.git] / drivers / net / virtio / virtio_rxtx.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <stdint.h>
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #include <errno.h>
39
40 #include <rte_cycles.h>
41 #include <rte_memory.h>
42 #include <rte_memzone.h>
43 #include <rte_branch_prediction.h>
44 #include <rte_mempool.h>
45 #include <rte_malloc.h>
46 #include <rte_mbuf.h>
47 #include <rte_ether.h>
48 #include <rte_ethdev.h>
49 #include <rte_prefetch.h>
50 #include <rte_string_fns.h>
51 #include <rte_errno.h>
52 #include <rte_byteorder.h>
53
54 #include "virtio_logs.h"
55 #include "virtio_ethdev.h"
56 #include "virtio_pci.h"
57 #include "virtqueue.h"
58 #include "virtio_rxtx.h"
59
60 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
61 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
62 #else
63 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
64 #endif
65
66
67 #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
68         ETH_TXQ_FLAGS_NOOFFLOADS)
69
70 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
71 static int use_simple_rxtx;
72 #endif
73
74 static void
75 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
76 {
77         struct vring_desc *dp, *dp_tail;
78         struct vq_desc_extra *dxp;
79         uint16_t desc_idx_last = desc_idx;
80
81         dp  = &vq->vq_ring.desc[desc_idx];
82         dxp = &vq->vq_descx[desc_idx];
83         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
84         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
85                 while (dp->flags & VRING_DESC_F_NEXT) {
86                         desc_idx_last = dp->next;
87                         dp = &vq->vq_ring.desc[dp->next];
88                 }
89         }
90         dxp->ndescs = 0;
91
92         /*
93          * We must append the existing free chain, if any, to the end of
94          * newly freed chain. If the virtqueue was completely used, then
95          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
96          */
97         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
98                 vq->vq_desc_head_idx = desc_idx;
99         } else {
100                 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
101                 dp_tail->next = desc_idx;
102         }
103
104         vq->vq_desc_tail_idx = desc_idx_last;
105         dp->next = VQ_RING_DESC_CHAIN_END;
106 }
107
108 static uint16_t
109 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
110                            uint32_t *len, uint16_t num)
111 {
112         struct vring_used_elem *uep;
113         struct rte_mbuf *cookie;
114         uint16_t used_idx, desc_idx;
115         uint16_t i;
116
117         /*  Caller does the check */
118         for (i = 0; i < num ; i++) {
119                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
120                 uep = &vq->vq_ring.used->ring[used_idx];
121                 desc_idx = (uint16_t) uep->id;
122                 len[i] = uep->len;
123                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
124
125                 if (unlikely(cookie == NULL)) {
126                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
127                                 vq->vq_used_cons_idx);
128                         break;
129                 }
130
131                 rte_prefetch0(cookie);
132                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
133                 rx_pkts[i]  = cookie;
134                 vq->vq_used_cons_idx++;
135                 vq_ring_free_chain(vq, desc_idx);
136                 vq->vq_descx[desc_idx].cookie = NULL;
137         }
138
139         return i;
140 }
141
142 #ifndef DEFAULT_TX_FREE_THRESH
143 #define DEFAULT_TX_FREE_THRESH 32
144 #endif
145
146 /* Cleanup from completed transmits. */
147 static void
148 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
149 {
150         uint16_t i, used_idx, desc_idx;
151         for (i = 0; i < num; i++) {
152                 struct vring_used_elem *uep;
153                 struct vq_desc_extra *dxp;
154
155                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
156                 uep = &vq->vq_ring.used->ring[used_idx];
157
158                 desc_idx = (uint16_t) uep->id;
159                 dxp = &vq->vq_descx[desc_idx];
160                 vq->vq_used_cons_idx++;
161                 vq_ring_free_chain(vq, desc_idx);
162
163                 if (dxp->cookie != NULL) {
164                         rte_pktmbuf_free(dxp->cookie);
165                         dxp->cookie = NULL;
166                 }
167         }
168 }
169
170
171 static inline int
172 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
173 {
174         struct vq_desc_extra *dxp;
175         struct virtio_hw *hw = vq->hw;
176         struct vring_desc *start_dp;
177         uint16_t needed = 1;
178         uint16_t head_idx, idx;
179
180         if (unlikely(vq->vq_free_cnt == 0))
181                 return -ENOSPC;
182         if (unlikely(vq->vq_free_cnt < needed))
183                 return -EMSGSIZE;
184
185         head_idx = vq->vq_desc_head_idx;
186         if (unlikely(head_idx >= vq->vq_nentries))
187                 return -EFAULT;
188
189         idx = head_idx;
190         dxp = &vq->vq_descx[idx];
191         dxp->cookie = (void *)cookie;
192         dxp->ndescs = needed;
193
194         start_dp = vq->vq_ring.desc;
195         start_dp[idx].addr =
196                 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM
197                 - hw->vtnet_hdr_size);
198         start_dp[idx].len =
199                 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
200         start_dp[idx].flags =  VRING_DESC_F_WRITE;
201         idx = start_dp[idx].next;
202         vq->vq_desc_head_idx = idx;
203         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
204                 vq->vq_desc_tail_idx = idx;
205         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
206         vq_update_avail_ring(vq, head_idx);
207
208         return 0;
209 }
210
211 static inline void
212 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
213                        uint16_t needed, int use_indirect, int can_push)
214 {
215         struct vq_desc_extra *dxp;
216         struct virtqueue *vq = txvq->vq;
217         struct vring_desc *start_dp;
218         uint16_t seg_num = cookie->nb_segs;
219         uint16_t head_idx, idx;
220         uint16_t head_size = vq->hw->vtnet_hdr_size;
221         unsigned long offs;
222
223         head_idx = vq->vq_desc_head_idx;
224         idx = head_idx;
225         dxp = &vq->vq_descx[idx];
226         dxp->cookie = (void *)cookie;
227         dxp->ndescs = needed;
228
229         start_dp = vq->vq_ring.desc;
230
231         if (can_push) {
232                 /* put on zero'd transmit header (no offloads) */
233                 void *hdr = rte_pktmbuf_prepend(cookie, head_size);
234
235                 memset(hdr, 0, head_size);
236         } else if (use_indirect) {
237                 /* setup tx ring slot to point to indirect
238                  * descriptor list stored in reserved region.
239                  *
240                  * the first slot in indirect ring is already preset
241                  * to point to the header in reserved region
242                  */
243                 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
244
245                 offs = idx * sizeof(struct virtio_tx_region)
246                         + offsetof(struct virtio_tx_region, tx_indir);
247
248                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem + offs;
249                 start_dp[idx].len   = (seg_num + 1) * sizeof(struct vring_desc);
250                 start_dp[idx].flags = VRING_DESC_F_INDIRECT;
251
252                 /* loop below will fill in rest of the indirect elements */
253                 start_dp = txr[idx].tx_indir;
254                 idx = 1;
255         } else {
256                 /* setup first tx ring slot to point to header
257                  * stored in reserved region.
258                  */
259                 offs = idx * sizeof(struct virtio_tx_region)
260                         + offsetof(struct virtio_tx_region, tx_hdr);
261
262                 start_dp[idx].addr  = txvq->virtio_net_hdr_mem + offs;
263                 start_dp[idx].len   = vq->hw->vtnet_hdr_size;
264                 start_dp[idx].flags = VRING_DESC_F_NEXT;
265                 idx = start_dp[idx].next;
266         }
267
268         do {
269                 start_dp[idx].addr  = rte_mbuf_data_dma_addr(cookie);
270                 start_dp[idx].len   = cookie->data_len;
271                 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0;
272                 idx = start_dp[idx].next;
273         } while ((cookie = cookie->next) != NULL);
274
275         if (use_indirect)
276                 idx = vq->vq_ring.desc[head_idx].next;
277
278         vq->vq_desc_head_idx = idx;
279         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
280                 vq->vq_desc_tail_idx = idx;
281         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
282         vq_update_avail_ring(vq, head_idx);
283 }
284
285 static void
286 virtio_dev_vring_start(struct virtqueue *vq)
287 {
288         int size = vq->vq_nentries;
289         struct vring *vr = &vq->vq_ring;
290         uint8_t *ring_mem = vq->vq_ring_virt_mem;
291
292         PMD_INIT_FUNC_TRACE();
293
294         /*
295          * Reinitialise since virtio port might have been stopped and restarted
296          */
297         memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size);
298         vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
299         vq->vq_used_cons_idx = 0;
300         vq->vq_desc_head_idx = 0;
301         vq->vq_avail_idx = 0;
302         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
303         vq->vq_free_cnt = vq->vq_nentries;
304         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
305
306         vring_desc_init(vr->desc, size);
307
308         /*
309          * Disable device(host) interrupting guest
310          */
311         virtqueue_disable_intr(vq);
312 }
313
314 void
315 virtio_dev_cq_start(struct rte_eth_dev *dev)
316 {
317         struct virtio_hw *hw = dev->data->dev_private;
318
319         if (hw->cvq && hw->cvq->vq) {
320                 virtio_dev_vring_start(hw->cvq->vq);
321                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq->vq);
322         }
323 }
324
325 void
326 virtio_dev_rxtx_start(struct rte_eth_dev *dev)
327 {
328         /*
329          * Start receive and transmit vrings
330          * -    Setup vring structure for all queues
331          * -    Initialize descriptor for the rx vring
332          * -    Allocate blank mbufs for the each rx descriptor
333          *
334          */
335         uint16_t i;
336         uint16_t desc_idx;
337
338         PMD_INIT_FUNC_TRACE();
339
340         /* Start rx vring. */
341         for (i = 0; i < dev->data->nb_rx_queues; i++) {
342                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
343                 struct virtqueue *vq = rxvq->vq;
344                 int error, nbufs;
345                 struct rte_mbuf *m;
346
347                 virtio_dev_vring_start(vq);
348                 if (rxvq->mpool == NULL) {
349                         rte_exit(EXIT_FAILURE,
350                                 "Cannot allocate mbufs for rx virtqueue");
351                 }
352
353                 /* Allocate blank mbufs for the each rx descriptor */
354                 nbufs = 0;
355                 error = ENOSPC;
356
357 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
358                 if (use_simple_rxtx) {
359                         for (desc_idx = 0; desc_idx < vq->vq_nentries;
360                              desc_idx++) {
361                                 vq->vq_ring.avail->ring[desc_idx] = desc_idx;
362                                 vq->vq_ring.desc[desc_idx].flags =
363                                         VRING_DESC_F_WRITE;
364                         }
365                 }
366 #endif
367                 memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf));
368                 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST;
369                      desc_idx++) {
370                         vq->sw_ring[vq->vq_nentries + desc_idx] =
371                                 &rxvq->fake_mbuf;
372                 }
373
374                 while (!virtqueue_full(vq)) {
375                         m = rte_mbuf_raw_alloc(rxvq->mpool);
376                         if (m == NULL)
377                                 break;
378
379                         /******************************************
380                         *         Enqueue allocated buffers        *
381                         *******************************************/
382 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
383                         if (use_simple_rxtx)
384                                 error = virtqueue_enqueue_recv_refill_simple(vq, m);
385                         else
386 #endif
387                                 error = virtqueue_enqueue_recv_refill(vq, m);
388                         if (error) {
389                                 rte_pktmbuf_free(m);
390                                 break;
391                         }
392                         nbufs++;
393                 }
394
395                 vq_update_avail_idx(vq);
396
397                 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
398
399                 VIRTQUEUE_DUMP(vq);
400         }
401
402         /* Start tx vring. */
403         for (i = 0; i < dev->data->nb_tx_queues; i++) {
404                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
405                 struct virtqueue *vq = txvq->vq;
406
407                 virtio_dev_vring_start(vq);
408 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
409                 if (use_simple_rxtx) {
410                         uint16_t mid_idx  = vq->vq_nentries >> 1;
411
412                         for (desc_idx = 0; desc_idx < mid_idx; desc_idx++) {
413                                 vq->vq_ring.avail->ring[desc_idx] =
414                                         desc_idx + mid_idx;
415                                 vq->vq_ring.desc[desc_idx + mid_idx].next =
416                                         desc_idx;
417                                 vq->vq_ring.desc[desc_idx + mid_idx].addr =
418                                         txvq->virtio_net_hdr_mem +
419                                         offsetof(struct virtio_tx_region, tx_hdr);
420                                 vq->vq_ring.desc[desc_idx + mid_idx].len =
421                                         vq->hw->vtnet_hdr_size;
422                                 vq->vq_ring.desc[desc_idx + mid_idx].flags =
423                                         VRING_DESC_F_NEXT;
424                                 vq->vq_ring.desc[desc_idx].flags = 0;
425                         }
426                         for (desc_idx = mid_idx; desc_idx < vq->vq_nentries;
427                              desc_idx++)
428                                 vq->vq_ring.avail->ring[desc_idx] = desc_idx;
429                 }
430 #endif
431                 VIRTQUEUE_DUMP(vq);
432         }
433 }
434
435 int
436 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
437                         uint16_t queue_idx,
438                         uint16_t nb_desc,
439                         unsigned int socket_id,
440                         __rte_unused const struct rte_eth_rxconf *rx_conf,
441                         struct rte_mempool *mp)
442 {
443         uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
444         struct virtnet_rx *rxvq;
445         int ret;
446
447         PMD_INIT_FUNC_TRACE();
448         ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
449                         nb_desc, socket_id, (void **)&rxvq);
450         if (ret < 0) {
451                 PMD_INIT_LOG(ERR, "rvq initialization failed");
452                 return ret;
453         }
454
455         /* Create mempool for rx mbuf allocation */
456         rxvq->mpool = mp;
457
458         dev->data->rx_queues[queue_idx] = rxvq;
459
460 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
461         virtio_rxq_vec_setup(rxvq);
462 #endif
463
464         return 0;
465 }
466
467 void
468 virtio_dev_rx_queue_release(void *rxq)
469 {
470         struct virtnet_rx *rxvq = rxq;
471         struct virtqueue *vq = rxvq->vq;
472         /* rxvq is freed when vq is freed, and as mz should be freed after the
473          * del_queue, so we reserve the mz pointer first.
474          */
475         const struct rte_memzone *mz = rxvq->mz;
476
477         /* no need to free rxq as vq and rxq are allocated together */
478         virtio_dev_queue_release(vq);
479         rte_memzone_free(mz);
480 }
481
482 /*
483  * struct rte_eth_dev *dev: Used to update dev
484  * uint16_t nb_desc: Defaults to values read from config space
485  * unsigned int socket_id: Used to allocate memzone
486  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
487  * uint16_t queue_idx: Just used as an index in dev txq list
488  */
489 int
490 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
491                         uint16_t queue_idx,
492                         uint16_t nb_desc,
493                         unsigned int socket_id,
494                         const struct rte_eth_txconf *tx_conf)
495 {
496         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
497
498 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
499         struct virtio_hw *hw = dev->data->dev_private;
500 #endif
501         struct virtnet_tx *txvq;
502         struct virtqueue *vq;
503         uint16_t tx_free_thresh;
504         int ret;
505
506         PMD_INIT_FUNC_TRACE();
507
508         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS)
509             != ETH_TXQ_FLAGS_NOXSUMS) {
510                 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n");
511                 return -EINVAL;
512         }
513
514 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
515         /* Use simple rx/tx func if single segment and no offloads */
516         if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS &&
517              !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
518                 PMD_INIT_LOG(INFO, "Using simple rx/tx path");
519                 dev->tx_pkt_burst = virtio_xmit_pkts_simple;
520                 dev->rx_pkt_burst = virtio_recv_pkts_vec;
521                 use_simple_rxtx = 1;
522         }
523 #endif
524
525         ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
526                         nb_desc, socket_id, (void **)&txvq);
527         if (ret < 0) {
528                 PMD_INIT_LOG(ERR, "tvq initialization failed");
529                 return ret;
530         }
531         vq = txvq->vq;
532
533         tx_free_thresh = tx_conf->tx_free_thresh;
534         if (tx_free_thresh == 0)
535                 tx_free_thresh =
536                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
537
538         if (tx_free_thresh >= (vq->vq_nentries - 3)) {
539                 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the "
540                         "number of TX entries minus 3 (%u)."
541                         " (tx_free_thresh=%u port=%u queue=%u)\n",
542                         vq->vq_nentries - 3,
543                         tx_free_thresh, dev->data->port_id, queue_idx);
544                 return -EINVAL;
545         }
546
547         vq->vq_free_thresh = tx_free_thresh;
548
549         dev->data->tx_queues[queue_idx] = txvq;
550         return 0;
551 }
552
553 void
554 virtio_dev_tx_queue_release(void *txq)
555 {
556         struct virtnet_tx *txvq = txq;
557         struct virtqueue *vq = txvq->vq;
558         /* txvq is freed when vq is freed, and as mz should be freed after the
559          * del_queue, so we reserve the mz pointer first.
560          */
561         const struct rte_memzone *hdr_mz = txvq->virtio_net_hdr_mz;
562         const struct rte_memzone *mz = txvq->mz;
563
564         virtio_dev_queue_release(vq);
565         rte_memzone_free(mz);
566         rte_memzone_free(hdr_mz);
567 }
568
569 static void
570 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
571 {
572         int error;
573         /*
574          * Requeue the discarded mbuf. This should always be
575          * successful since it was just dequeued.
576          */
577         error = virtqueue_enqueue_recv_refill(vq, m);
578         if (unlikely(error)) {
579                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
580                 rte_pktmbuf_free(m);
581         }
582 }
583
584 static void
585 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
586 {
587         uint32_t s = mbuf->pkt_len;
588         struct ether_addr *ea;
589
590         if (s == 64) {
591                 stats->size_bins[1]++;
592         } else if (s > 64 && s < 1024) {
593                 uint32_t bin;
594
595                 /* count zeros, and offset into correct bin */
596                 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
597                 stats->size_bins[bin]++;
598         } else {
599                 if (s < 64)
600                         stats->size_bins[0]++;
601                 else if (s < 1519)
602                         stats->size_bins[6]++;
603                 else if (s >= 1519)
604                         stats->size_bins[7]++;
605         }
606
607         ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
608         if (is_multicast_ether_addr(ea)) {
609                 if (is_broadcast_ether_addr(ea))
610                         stats->broadcast++;
611                 else
612                         stats->multicast++;
613         }
614 }
615
616 #define VIRTIO_MBUF_BURST_SZ 64
617 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
618 uint16_t
619 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
620 {
621         struct virtnet_rx *rxvq = rx_queue;
622         struct virtqueue *vq = rxvq->vq;
623         struct virtio_hw *hw;
624         struct rte_mbuf *rxm, *new_mbuf;
625         uint16_t nb_used, num, nb_rx;
626         uint32_t len[VIRTIO_MBUF_BURST_SZ];
627         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
628         int error;
629         uint32_t i, nb_enqueued;
630         uint32_t hdr_size;
631
632         nb_used = VIRTQUEUE_NUSED(vq);
633
634         virtio_rmb();
635
636         num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
637         num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
638         if (likely(num > DESC_PER_CACHELINE))
639                 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
640
641         num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
642         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
643
644         hw = vq->hw;
645         nb_rx = 0;
646         nb_enqueued = 0;
647         hdr_size = hw->vtnet_hdr_size;
648
649         for (i = 0; i < num ; i++) {
650                 rxm = rcv_pkts[i];
651
652                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
653
654                 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
655                         PMD_RX_LOG(ERR, "Packet drop");
656                         nb_enqueued++;
657                         virtio_discard_rxbuf(vq, rxm);
658                         rxvq->stats.errors++;
659                         continue;
660                 }
661
662                 rxm->port = rxvq->port_id;
663                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
664                 rxm->ol_flags = 0;
665                 rxm->vlan_tci = 0;
666
667                 rxm->nb_segs = 1;
668                 rxm->next = NULL;
669                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
670                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
671
672                 if (hw->vlan_strip)
673                         rte_vlan_strip(rxm);
674
675                 VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
676
677                 rx_pkts[nb_rx++] = rxm;
678
679                 rxvq->stats.bytes += rx_pkts[nb_rx - 1]->pkt_len;
680                 virtio_update_packet_stats(&rxvq->stats, rxm);
681         }
682
683         rxvq->stats.packets += nb_rx;
684
685         /* Allocate new mbuf for the used descriptor */
686         error = ENOSPC;
687         while (likely(!virtqueue_full(vq))) {
688                 new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
689                 if (unlikely(new_mbuf == NULL)) {
690                         struct rte_eth_dev *dev
691                                 = &rte_eth_devices[rxvq->port_id];
692                         dev->data->rx_mbuf_alloc_failed++;
693                         break;
694                 }
695                 error = virtqueue_enqueue_recv_refill(vq, new_mbuf);
696                 if (unlikely(error)) {
697                         rte_pktmbuf_free(new_mbuf);
698                         break;
699                 }
700                 nb_enqueued++;
701         }
702
703         if (likely(nb_enqueued)) {
704                 vq_update_avail_idx(vq);
705
706                 if (unlikely(virtqueue_kick_prepare(vq))) {
707                         virtqueue_notify(vq);
708                         PMD_RX_LOG(DEBUG, "Notified");
709                 }
710         }
711
712         return nb_rx;
713 }
714
715 uint16_t
716 virtio_recv_mergeable_pkts(void *rx_queue,
717                         struct rte_mbuf **rx_pkts,
718                         uint16_t nb_pkts)
719 {
720         struct virtnet_rx *rxvq = rx_queue;
721         struct virtqueue *vq = rxvq->vq;
722         struct virtio_hw *hw;
723         struct rte_mbuf *rxm, *new_mbuf;
724         uint16_t nb_used, num, nb_rx;
725         uint32_t len[VIRTIO_MBUF_BURST_SZ];
726         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
727         struct rte_mbuf *prev;
728         int error;
729         uint32_t i, nb_enqueued;
730         uint32_t seg_num;
731         uint16_t extra_idx;
732         uint32_t seg_res;
733         uint32_t hdr_size;
734
735         nb_used = VIRTQUEUE_NUSED(vq);
736
737         virtio_rmb();
738
739         PMD_RX_LOG(DEBUG, "used:%d", nb_used);
740
741         hw = vq->hw;
742         nb_rx = 0;
743         i = 0;
744         nb_enqueued = 0;
745         seg_num = 0;
746         extra_idx = 0;
747         seg_res = 0;
748         hdr_size = hw->vtnet_hdr_size;
749
750         while (i < nb_used) {
751                 struct virtio_net_hdr_mrg_rxbuf *header;
752
753                 if (nb_rx == nb_pkts)
754                         break;
755
756                 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1);
757                 if (num != 1)
758                         continue;
759
760                 i++;
761
762                 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
763                 PMD_RX_LOG(DEBUG, "packet len:%d", len[0]);
764
765                 rxm = rcv_pkts[0];
766
767                 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
768                         PMD_RX_LOG(ERR, "Packet drop");
769                         nb_enqueued++;
770                         virtio_discard_rxbuf(vq, rxm);
771                         rxvq->stats.errors++;
772                         continue;
773                 }
774
775                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
776                         RTE_PKTMBUF_HEADROOM - hdr_size);
777                 seg_num = header->num_buffers;
778
779                 if (seg_num == 0)
780                         seg_num = 1;
781
782                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
783                 rxm->nb_segs = seg_num;
784                 rxm->next = NULL;
785                 rxm->ol_flags = 0;
786                 rxm->vlan_tci = 0;
787                 rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
788                 rxm->data_len = (uint16_t)(len[0] - hdr_size);
789
790                 rxm->port = rxvq->port_id;
791                 rx_pkts[nb_rx] = rxm;
792                 prev = rxm;
793
794                 seg_res = seg_num - 1;
795
796                 while (seg_res != 0) {
797                         /*
798                          * Get extra segments for current uncompleted packet.
799                          */
800                         uint16_t  rcv_cnt =
801                                 RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
802                         if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) {
803                                 uint32_t rx_num =
804                                         virtqueue_dequeue_burst_rx(vq,
805                                         rcv_pkts, len, rcv_cnt);
806                                 i += rx_num;
807                                 rcv_cnt = rx_num;
808                         } else {
809                                 PMD_RX_LOG(ERR,
810                                            "No enough segments for packet.");
811                                 nb_enqueued++;
812                                 virtio_discard_rxbuf(vq, rxm);
813                                 rxvq->stats.errors++;
814                                 break;
815                         }
816
817                         extra_idx = 0;
818
819                         while (extra_idx < rcv_cnt) {
820                                 rxm = rcv_pkts[extra_idx];
821
822                                 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
823                                 rxm->next = NULL;
824                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
825                                 rxm->data_len = (uint16_t)(len[extra_idx]);
826
827                                 if (prev)
828                                         prev->next = rxm;
829
830                                 prev = rxm;
831                                 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len;
832                                 extra_idx++;
833                         };
834                         seg_res -= rcv_cnt;
835                 }
836
837                 if (hw->vlan_strip)
838                         rte_vlan_strip(rx_pkts[nb_rx]);
839
840                 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
841                         rx_pkts[nb_rx]->data_len);
842
843                 rxvq->stats.bytes += rx_pkts[nb_rx]->pkt_len;
844                 virtio_update_packet_stats(&rxvq->stats, rx_pkts[nb_rx]);
845                 nb_rx++;
846         }
847
848         rxvq->stats.packets += nb_rx;
849
850         /* Allocate new mbuf for the used descriptor */
851         error = ENOSPC;
852         while (likely(!virtqueue_full(vq))) {
853                 new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
854                 if (unlikely(new_mbuf == NULL)) {
855                         struct rte_eth_dev *dev
856                                 = &rte_eth_devices[rxvq->port_id];
857                         dev->data->rx_mbuf_alloc_failed++;
858                         break;
859                 }
860                 error = virtqueue_enqueue_recv_refill(vq, new_mbuf);
861                 if (unlikely(error)) {
862                         rte_pktmbuf_free(new_mbuf);
863                         break;
864                 }
865                 nb_enqueued++;
866         }
867
868         if (likely(nb_enqueued)) {
869                 vq_update_avail_idx(vq);
870
871                 if (unlikely(virtqueue_kick_prepare(vq))) {
872                         virtqueue_notify(vq);
873                         PMD_RX_LOG(DEBUG, "Notified");
874                 }
875         }
876
877         return nb_rx;
878 }
879
880 uint16_t
881 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
882 {
883         struct virtnet_tx *txvq = tx_queue;
884         struct virtqueue *vq = txvq->vq;
885         struct virtio_hw *hw = vq->hw;
886         uint16_t hdr_size = hw->vtnet_hdr_size;
887         uint16_t nb_used, nb_tx;
888         int error;
889
890         if (unlikely(nb_pkts < 1))
891                 return nb_pkts;
892
893         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
894         nb_used = VIRTQUEUE_NUSED(vq);
895
896         virtio_rmb();
897         if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
898                 virtio_xmit_cleanup(vq, nb_used);
899
900         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
901                 struct rte_mbuf *txm = tx_pkts[nb_tx];
902                 int can_push = 0, use_indirect = 0, slots, need;
903
904                 /* Do VLAN tag insertion */
905                 if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
906                         error = rte_vlan_insert(&txm);
907                         if (unlikely(error)) {
908                                 rte_pktmbuf_free(txm);
909                                 continue;
910                         }
911                 }
912
913                 /* optimize ring usage */
914                 if (vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) &&
915                     rte_mbuf_refcnt_read(txm) == 1 &&
916                     RTE_MBUF_DIRECT(txm) &&
917                     txm->nb_segs == 1 &&
918                     rte_pktmbuf_headroom(txm) >= hdr_size &&
919                     rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
920                                    __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
921                         can_push = 1;
922                 else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
923                          txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
924                         use_indirect = 1;
925
926                 /* How many main ring entries are needed to this Tx?
927                  * any_layout => number of segments
928                  * indirect   => 1
929                  * default    => number of segments + 1
930                  */
931                 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
932                 need = slots - vq->vq_free_cnt;
933
934                 /* Positive value indicates it need free vring descriptors */
935                 if (unlikely(need > 0)) {
936                         nb_used = VIRTQUEUE_NUSED(vq);
937                         virtio_rmb();
938                         need = RTE_MIN(need, (int)nb_used);
939
940                         virtio_xmit_cleanup(vq, need);
941                         need = slots - vq->vq_free_cnt;
942                         if (unlikely(need > 0)) {
943                                 PMD_TX_LOG(ERR,
944                                            "No free tx descriptors to transmit");
945                                 break;
946                         }
947                 }
948
949                 /* Enqueue Packet buffers */
950                 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, can_push);
951
952                 txvq->stats.bytes += txm->pkt_len;
953                 virtio_update_packet_stats(&txvq->stats, txm);
954         }
955
956         txvq->stats.packets += nb_tx;
957
958         if (likely(nb_tx)) {
959                 vq_update_avail_idx(vq);
960
961                 if (unlikely(virtqueue_kick_prepare(vq))) {
962                         virtqueue_notify(vq);
963                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
964                 }
965         }
966
967         return nb_tx;
968 }