lib/librte_pmd_virtio/virtio_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 #include <rte_cycles.h>
  41 #include <rte_memory.h>
  42 #include <rte_memzone.h>
  43 #include <rte_branch_prediction.h>
  44 #include <rte_mempool.h>
  45 #include <rte_malloc.h>
  46 #include <rte_mbuf.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_prefetch.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_errno.h>
  52
  53 #include "virtio_logs.h"
  54 #include "virtio_ethdev.h"
  55 #include "virtqueue.h"
  56
  57 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
  58 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
  59 #else
  60 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
  61 #endif
  62
  63 static void
  64 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  65 {
  66         struct vring_desc *dp, *dp_tail;
  67         struct vq_desc_extra *dxp;
  68         uint16_t desc_idx_last = desc_idx;
  69
  70         dp  = &vq->vq_ring.desc[desc_idx];
  71         dxp = &vq->vq_descx[desc_idx];
  72         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
  73         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
  74                 while (dp->flags & VRING_DESC_F_NEXT) {
  75                         desc_idx_last = dp->next;
  76                         dp = &vq->vq_ring.desc[dp->next];
  77                 }
  78         }
  79         dxp->ndescs = 0;
  80
  81         /*
  82          * We must append the existing free chain, if any, to the end of
  83          * newly freed chain. If the virtqueue was completely used, then
  84          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  85          */
  86         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
  87                 vq->vq_desc_head_idx = desc_idx;
  88         } else {
  89                 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
  90                 dp_tail->next = desc_idx;
  91         }
  92
  93         vq->vq_desc_tail_idx = desc_idx_last;
  94         dp->next = VQ_RING_DESC_CHAIN_END;
  95 }
  96
  97 static uint16_t
  98 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
  99                            uint32_t *len, uint16_t num)
 100 {
 101         struct vring_used_elem *uep;
 102         struct rte_mbuf *cookie;
 103         uint16_t used_idx, desc_idx;
 104         uint16_t i;
 105
 106         /*  Caller does the check */
 107         for (i = 0; i < num ; i++) {
 108                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 109                 uep = &vq->vq_ring.used->ring[used_idx];
 110                 desc_idx = (uint16_t) uep->id;
 111                 len[i] = uep->len;
 112                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
 113
 114                 if (unlikely(cookie == NULL)) {
 115                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
 116                                 vq->vq_used_cons_idx);
 117                         break;
 118                 }
 119
 120                 rte_prefetch0(cookie);
 121                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 122                 rx_pkts[i]  = cookie;
 123                 vq->vq_used_cons_idx++;
 124                 vq_ring_free_chain(vq, desc_idx);
 125                 vq->vq_descx[desc_idx].cookie = NULL;
 126         }
 127
 128         return i;
 129 }
 130
 131 static void
 132 virtqueue_dequeue_pkt_tx(struct virtqueue *vq)
 133 {
 134         struct vring_used_elem *uep;
 135         uint16_t used_idx, desc_idx;
 136
 137         used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 138         uep = &vq->vq_ring.used->ring[used_idx];
 139         desc_idx = (uint16_t) uep->id;
 140         vq->vq_used_cons_idx++;
 141         vq_ring_free_chain(vq, desc_idx);
 142 }
 143
 144
 145 static inline int
 146 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
 147 {
 148         struct vq_desc_extra *dxp;
 149         struct virtio_hw *hw = vq->hw;
 150         struct vring_desc *start_dp;
 151         uint16_t needed = 1;
 152         uint16_t head_idx, idx;
 153
 154         if (unlikely(vq->vq_free_cnt == 0))
 155                 return -ENOSPC;
 156         if (unlikely(vq->vq_free_cnt < needed))
 157                 return -EMSGSIZE;
 158
 159         head_idx = vq->vq_desc_head_idx;
 160         if (unlikely(head_idx >= vq->vq_nentries))
 161                 return -EFAULT;
 162
 163         idx = head_idx;
 164         dxp = &vq->vq_descx[idx];
 165         dxp->cookie = (void *)cookie;
 166         dxp->ndescs = needed;
 167
 168         start_dp = vq->vq_ring.desc;
 169         start_dp[idx].addr =
 170                 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM
 171                 - hw->vtnet_hdr_size);
 172         start_dp[idx].len =
 173                 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 174         start_dp[idx].flags =  VRING_DESC_F_WRITE;
 175         idx = start_dp[idx].next;
 176         vq->vq_desc_head_idx = idx;
 177         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 178                 vq->vq_desc_tail_idx = idx;
 179         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 180         vq_update_avail_ring(vq, head_idx);
 181
 182         return 0;
 183 }
 184
 185 static int
 186 virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
 187 {
 188         struct vq_desc_extra *dxp;
 189         struct vring_desc *start_dp;
 190         uint16_t seg_num = cookie->nb_segs;
 191         uint16_t needed = 1 + seg_num;
 192         uint16_t head_idx, idx;
 193         uint16_t head_size = txvq->hw->vtnet_hdr_size;
 194
 195         if (unlikely(txvq->vq_free_cnt == 0))
 196                 return -ENOSPC;
 197         if (unlikely(txvq->vq_free_cnt < needed))
 198                 return -EMSGSIZE;
 199         head_idx = txvq->vq_desc_head_idx;
 200         if (unlikely(head_idx >= txvq->vq_nentries))
 201                 return -EFAULT;
 202
 203         idx = head_idx;
 204         dxp = &txvq->vq_descx[idx];
 205         if (dxp->cookie != NULL)
 206                 rte_pktmbuf_free(dxp->cookie);
 207         dxp->cookie = (void *)cookie;
 208         dxp->ndescs = needed;
 209
 210         start_dp = txvq->vq_ring.desc;
 211         start_dp[idx].addr =
 212                 txvq->virtio_net_hdr_mem + idx * head_size;
 213         start_dp[idx].len = (uint32_t)head_size;
 214         start_dp[idx].flags = VRING_DESC_F_NEXT;
 215
 216         for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) {
 217                 idx = start_dp[idx].next;
 218                 start_dp[idx].addr  = RTE_MBUF_DATA_DMA_ADDR(cookie);
 219                 start_dp[idx].len   = cookie->data_len;
 220                 start_dp[idx].flags = VRING_DESC_F_NEXT;
 221                 cookie = cookie->next;
 222         }
 223
 224         start_dp[idx].flags &= ~VRING_DESC_F_NEXT;
 225         idx = start_dp[idx].next;
 226         txvq->vq_desc_head_idx = idx;
 227         if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 228                 txvq->vq_desc_tail_idx = idx;
 229         txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
 230         vq_update_avail_ring(txvq, head_idx);
 231
 232         return 0;
 233 }
 234
 235 static inline struct rte_mbuf *
 236 rte_rxmbuf_alloc(struct rte_mempool *mp)
 237 {
 238         struct rte_mbuf *m;
 239
 240         m = __rte_mbuf_raw_alloc(mp);
 241         __rte_mbuf_sanity_check_raw(m, 0);
 242
 243         return m;
 244 }
 245
 246 static void
 247 virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
 248 {
 249         struct rte_mbuf *m;
 250         int i, nbufs, error, size = vq->vq_nentries;
 251         struct vring *vr = &vq->vq_ring;
 252         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 253
 254         PMD_INIT_FUNC_TRACE();
 255
 256         /*
 257          * Reinitialise since virtio port might have been stopped and restarted
 258          */
 259         memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size);
 260         vring_init(vr, size, ring_mem, vq->vq_alignment);
 261         vq->vq_used_cons_idx = 0;
 262         vq->vq_desc_head_idx = 0;
 263         vq->vq_avail_idx = 0;
 264         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 265         vq->vq_free_cnt = vq->vq_nentries;
 266         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 267
 268         /* Chain all the descriptors in the ring with an END */
 269         for (i = 0; i < size - 1; i++)
 270                 vr->desc[i].next = (uint16_t)(i + 1);
 271         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 272
 273         /*
 274          * Disable device(host) interrupting guest
 275          */
 276         virtqueue_disable_intr(vq);
 277
 278         /* Only rx virtqueue needs mbufs to be allocated at initialization */
 279         if (queue_type == VTNET_RQ) {
 280                 if (vq->mpool == NULL)
 281                         rte_exit(EXIT_FAILURE,
 282                         "Cannot allocate initial mbufs for rx virtqueue");
 283
 284                 /* Allocate blank mbufs for the each rx descriptor */
 285                 nbufs = 0;
 286                 error = ENOSPC;
 287                 while (!virtqueue_full(vq)) {
 288                         m = rte_rxmbuf_alloc(vq->mpool);
 289                         if (m == NULL)
 290                                 break;
 291
 292                         /******************************************
 293                         *         Enqueue allocated buffers        *
 294                         *******************************************/
 295                         error = virtqueue_enqueue_recv_refill(vq, m);
 296
 297                         if (error) {
 298                                 rte_pktmbuf_free(m);
 299                                 break;
 300                         }
 301                         nbufs++;
 302                 }
 303
 304                 vq_update_avail_idx(vq);
 305
 306                 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
 307
 308                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 309                         vq->vq_queue_index);
 310                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 311                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 312         } else if (queue_type == VTNET_TQ) {
 313                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 314                         vq->vq_queue_index);
 315                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 316                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 317         } else {
 318                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 319                         vq->vq_queue_index);
 320                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 321                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 322         }
 323 }
 324
 325 void
 326 virtio_dev_cq_start(struct rte_eth_dev *dev)
 327 {
 328         struct virtio_hw *hw
 329                 = VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 330
 331         if (hw->cvq) {
 332                 virtio_dev_vring_start(hw->cvq, VTNET_CQ);
 333                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
 334         }
 335 }
 336
 337 void
 338 virtio_dev_rxtx_start(struct rte_eth_dev *dev)
 339 {
 340         /*
 341          * Start receive and transmit vrings
 342          * -    Setup vring structure for all queues
 343          * -    Initialize descriptor for the rx vring
 344          * -    Allocate blank mbufs for the each rx descriptor
 345          *
 346          */
 347         int i;
 348
 349         PMD_INIT_FUNC_TRACE();
 350
 351         /* Start rx vring. */
 352         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 353                 virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ);
 354                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 355         }
 356
 357         /* Start tx vring. */
 358         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 359                 virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ);
 360                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 361         }
 362 }
 363
 364 int
 365 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 366                         uint16_t queue_idx,
 367                         uint16_t nb_desc,
 368                         unsigned int socket_id,
 369                         __rte_unused const struct rte_eth_rxconf *rx_conf,
 370                         struct rte_mempool *mp)
 371 {
 372         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 373         struct virtqueue *vq;
 374         int ret;
 375
 376         PMD_INIT_FUNC_TRACE();
 377         ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
 378                         nb_desc, socket_id, &vq);
 379         if (ret < 0) {
 380                 PMD_INIT_LOG(ERR, "tvq initialization failed");
 381                 return ret;
 382         }
 383
 384         /* Create mempool for rx mbuf allocation */
 385         vq->mpool = mp;
 386
 387         dev->data->rx_queues[queue_idx] = vq;
 388         return 0;
 389 }
 390
 391 /*
 392  * struct rte_eth_dev *dev: Used to update dev
 393  * uint16_t nb_desc: Defaults to values read from config space
 394  * unsigned int socket_id: Used to allocate memzone
 395  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
 396  * uint16_t queue_idx: Just used as an index in dev txq list
 397  */
 398 int
 399 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 400                         uint16_t queue_idx,
 401                         uint16_t nb_desc,
 402                         unsigned int socket_id,
 403                         const struct rte_eth_txconf *tx_conf)
 404 {
 405         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 406         struct virtqueue *vq;
 407         int ret;
 408
 409         PMD_INIT_FUNC_TRACE();
 410
 411         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOOFFLOADS)
 412             != ETH_TXQ_FLAGS_NOOFFLOADS) {
 413                 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n");
 414                 return -EINVAL;
 415         }
 416
 417         ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
 418                         nb_desc, socket_id, &vq);
 419         if (ret < 0) {
 420                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 421                 return ret;
 422         }
 423
 424         dev->data->tx_queues[queue_idx] = vq;
 425         return 0;
 426 }
 427
 428 static void
 429 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 430 {
 431         int error;
 432         /*
 433          * Requeue the discarded mbuf. This should always be
 434          * successful since it was just dequeued.
 435          */
 436         error = virtqueue_enqueue_recv_refill(vq, m);
 437         if (unlikely(error)) {
 438                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
 439                 rte_pktmbuf_free(m);
 440         }
 441 }
 442
 443 #define VIRTIO_MBUF_BURST_SZ 64
 444 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 445 uint16_t
 446 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 447 {
 448         struct virtqueue *rxvq = rx_queue;
 449         struct rte_mbuf *rxm, *new_mbuf;
 450         uint16_t nb_used, num, nb_rx = 0;
 451         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 452         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 453         int error;
 454         uint32_t i, nb_enqueued = 0;
 455         const uint32_t hdr_size = sizeof(struct virtio_net_hdr);
 456
 457         nb_used = VIRTQUEUE_NUSED(rxvq);
 458
 459         rmb();
 460
 461         num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
 462         num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
 463         if (likely(num > DESC_PER_CACHELINE))
 464                 num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
 465
 466         if (num == 0)
 467                 return 0;
 468
 469         num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num);
 470         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
 471         for (i = 0; i < num ; i++) {
 472                 rxm = rcv_pkts[i];
 473
 474                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
 475
 476                 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
 477                         PMD_RX_LOG(ERR, "Packet drop");
 478                         nb_enqueued++;
 479                         virtio_discard_rxbuf(rxvq, rxm);
 480                         rxvq->errors++;
 481                         continue;
 482                 }
 483
 484                 rxm->port = rxvq->port_id;
 485                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 486
 487                 rxm->nb_segs = 1;
 488                 rxm->next = NULL;
 489                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
 490                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
 491
 492                 VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
 493
 494                 rx_pkts[nb_rx++] = rxm;
 495                 rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
 496         }
 497
 498         rxvq->packets += nb_rx;
 499
 500         /* Allocate new mbuf for the used descriptor */
 501         error = ENOSPC;
 502         while (likely(!virtqueue_full(rxvq))) {
 503                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 504                 if (unlikely(new_mbuf == NULL)) {
 505                         struct rte_eth_dev *dev
 506                                 = &rte_eth_devices[rxvq->port_id];
 507                         dev->data->rx_mbuf_alloc_failed++;
 508                         break;
 509                 }
 510                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 511                 if (unlikely(error)) {
 512                         rte_pktmbuf_free(new_mbuf);
 513                         break;
 514                 }
 515                 nb_enqueued++;
 516         }
 517
 518         if (likely(nb_enqueued)) {
 519                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 520                         virtqueue_notify(rxvq);
 521                         PMD_RX_LOG(DEBUG, "Notified\n");
 522                 }
 523         }
 524
 525         vq_update_avail_idx(rxvq);
 526
 527         return nb_rx;
 528 }
 529
 530 uint16_t
 531 virtio_recv_mergeable_pkts(void *rx_queue,
 532                         struct rte_mbuf **rx_pkts,
 533                         uint16_t nb_pkts)
 534 {
 535         struct virtqueue *rxvq = rx_queue;
 536         struct rte_mbuf *rxm, *new_mbuf;
 537         uint16_t nb_used, num, nb_rx = 0;
 538         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 539         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 540         struct rte_mbuf *prev;
 541         int error;
 542         uint32_t i = 0, nb_enqueued = 0;
 543         uint32_t seg_num = 0;
 544         uint16_t extra_idx = 0;
 545         uint32_t seg_res = 0;
 546         const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 547
 548         nb_used = VIRTQUEUE_NUSED(rxvq);
 549
 550         rmb();
 551
 552         if (nb_used == 0)
 553                 return 0;
 554
 555         PMD_RX_LOG(DEBUG, "used:%d\n", nb_used);
 556
 557         while (i < nb_used) {
 558                 struct virtio_net_hdr_mrg_rxbuf *header;
 559
 560                 if (nb_rx == nb_pkts)
 561                         break;
 562
 563                 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1);
 564                 if (num != 1)
 565                         continue;
 566
 567                 i++;
 568
 569                 PMD_RX_LOG(DEBUG, "dequeue:%d\n", num);
 570                 PMD_RX_LOG(DEBUG, "packet len:%d\n", len[0]);
 571
 572                 rxm = rcv_pkts[0];
 573
 574                 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
 575                         PMD_RX_LOG(ERR, "Packet drop\n");
 576                         nb_enqueued++;
 577                         virtio_discard_rxbuf(rxvq, rxm);
 578                         rxvq->errors++;
 579                         continue;
 580                 }
 581
 582                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
 583                         RTE_PKTMBUF_HEADROOM - hdr_size);
 584                 seg_num = header->num_buffers;
 585
 586                 if (seg_num == 0)
 587                         seg_num = 1;
 588
 589                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 590                 rxm->nb_segs = seg_num;
 591                 rxm->next = NULL;
 592                 rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
 593                 rxm->data_len = (uint16_t)(len[0] - hdr_size);
 594
 595                 rxm->port = rxvq->port_id;
 596                 rx_pkts[nb_rx] = rxm;
 597                 prev = rxm;
 598
 599                 seg_res = seg_num - 1;
 600
 601                 while (seg_res != 0) {
 602                         /*
 603                          * Get extra segments for current uncompleted packet.
 604                          */
 605                         uint32_t  rcv_cnt =
 606                                 RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
 607                         if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) {
 608                                 uint32_t rx_num =
 609                                         virtqueue_dequeue_burst_rx(rxvq,
 610                                         rcv_pkts, len, rcv_cnt);
 611                                 i += rx_num;
 612                                 rcv_cnt = rx_num;
 613                         } else {
 614                                 PMD_RX_LOG(ERR,
 615                                         "No enough segments for packet.\n");
 616                                 nb_enqueued++;
 617                                 virtio_discard_rxbuf(rxvq, rxm);
 618                                 rxvq->errors++;
 619                                 break;
 620                         }
 621
 622                         extra_idx = 0;
 623
 624                         while (extra_idx < rcv_cnt) {
 625                                 rxm = rcv_pkts[extra_idx];
 626
 627                                 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
 628                                 rxm->next = NULL;
 629                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
 630                                 rxm->data_len = (uint16_t)(len[extra_idx]);
 631
 632                                 if (prev)
 633                                         prev->next = rxm;
 634
 635                                 prev = rxm;
 636                                 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len;
 637                                 extra_idx++;
 638                         };
 639                         seg_res -= rcv_cnt;
 640                 }
 641
 642                 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
 643                         rx_pkts[nb_rx]->data_len);
 644
 645                 rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
 646                 nb_rx++;
 647         }
 648
 649         rxvq->packets += nb_rx;
 650
 651         /* Allocate new mbuf for the used descriptor */
 652         error = ENOSPC;
 653         while (likely(!virtqueue_full(rxvq))) {
 654                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 655                 if (unlikely(new_mbuf == NULL)) {
 656                         struct rte_eth_dev *dev
 657                                 = &rte_eth_devices[rxvq->port_id];
 658                         dev->data->rx_mbuf_alloc_failed++;
 659                         break;
 660                 }
 661                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 662                 if (unlikely(error)) {
 663                         rte_pktmbuf_free(new_mbuf);
 664                         break;
 665                 }
 666                 nb_enqueued++;
 667         }
 668
 669         if (likely(nb_enqueued)) {
 670                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 671                         virtqueue_notify(rxvq);
 672                         PMD_RX_LOG(DEBUG, "Notified");
 673                 }
 674         }
 675
 676         vq_update_avail_idx(rxvq);
 677
 678         return nb_rx;
 679 }
 680
 681 uint16_t
 682 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 683 {
 684         struct virtqueue *txvq = tx_queue;
 685         struct rte_mbuf *txm;
 686         uint16_t nb_used, nb_tx, num;
 687         int error;
 688
 689         nb_tx = 0;
 690
 691         if (unlikely(nb_pkts < 1))
 692                 return nb_pkts;
 693
 694         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 695         nb_used = VIRTQUEUE_NUSED(txvq);
 696
 697         rmb();
 698
 699         num = (uint16_t)(likely(nb_used < VIRTIO_MBUF_BURST_SZ) ? nb_used : VIRTIO_MBUF_BURST_SZ);
 700
 701         while (nb_tx < nb_pkts) {
 702                 /* Need one more descriptor for virtio header. */
 703                 int need = tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1;
 704                 int deq_cnt = RTE_MIN(need, (int)num);
 705
 706                 num -= (deq_cnt > 0) ? deq_cnt : 0;
 707                 while (deq_cnt > 0) {
 708                         virtqueue_dequeue_pkt_tx(txvq);
 709                         deq_cnt--;
 710                 }
 711
 712                 need = (int)tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1;
 713                 /*
 714                  * Zero or negative value indicates it has enough free
 715                  * descriptors to use for transmitting.
 716                  */
 717                 if (likely(need <= 0)) {
 718                         txm = tx_pkts[nb_tx];
 719                         /* Enqueue Packet buffers */
 720                         error = virtqueue_enqueue_xmit(txvq, txm);
 721                         if (unlikely(error)) {
 722                                 if (error == ENOSPC)
 723                                         PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0");
 724                                 else if (error == EMSGSIZE)
 725                                         PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1");
 726                                 else
 727                                         PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error);
 728                                 break;
 729                         }
 730                         nb_tx++;
 731                         txvq->bytes += txm->pkt_len;
 732                 } else {
 733                         PMD_TX_LOG(ERR, "No free tx descriptors to transmit");
 734                         break;
 735                 }
 736         }
 737         vq_update_avail_idx(txvq);
 738
 739         txvq->packets += nb_tx;
 740
 741         if (unlikely(virtqueue_kick_prepare(txvq))) {
 742                 virtqueue_notify(txvq);
 743                 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
 744         }
 745
 746         return nb_tx;
 747 }