lib/librte_pmd_virtio/virtio_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 #include <rte_cycles.h>
  41 #include <rte_memory.h>
  42 #include <rte_memzone.h>
  43 #include <rte_branch_prediction.h>
  44 #include <rte_mempool.h>
  45 #include <rte_malloc.h>
  46 #include <rte_mbuf.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_prefetch.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_errno.h>
  52
  53 #include "virtio_logs.h"
  54 #include "virtio_ethdev.h"
  55 #include "virtqueue.h"
  56
  57 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
  58 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
  59 #else
  60 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
  61 #endif
  62
  63 static void
  64 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  65 {
  66         struct vring_desc *dp, *dp_tail;
  67         struct vq_desc_extra *dxp;
  68         uint16_t desc_idx_last = desc_idx;
  69
  70         dp  = &vq->vq_ring.desc[desc_idx];
  71         dxp = &vq->vq_descx[desc_idx];
  72         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
  73         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
  74                 while (dp->flags & VRING_DESC_F_NEXT) {
  75                         desc_idx_last = dp->next;
  76                         dp = &vq->vq_ring.desc[dp->next];
  77                 }
  78         }
  79         dxp->ndescs = 0;
  80
  81         /*
  82          * We must append the existing free chain, if any, to the end of
  83          * newly freed chain. If the virtqueue was completely used, then
  84          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  85          */
  86         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
  87                 vq->vq_desc_head_idx = desc_idx;
  88         } else {
  89                 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
  90                 dp_tail->next = desc_idx;
  91         }
  92
  93         vq->vq_desc_tail_idx = desc_idx_last;
  94         dp->next = VQ_RING_DESC_CHAIN_END;
  95 }
  96
  97 static uint16_t
  98 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
  99                            uint32_t *len, uint16_t num)
 100 {
 101         struct vring_used_elem *uep;
 102         struct rte_mbuf *cookie;
 103         uint16_t used_idx, desc_idx;
 104         uint16_t i;
 105
 106         /*  Caller does the check */
 107         for (i = 0; i < num ; i++) {
 108                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 109                 uep = &vq->vq_ring.used->ring[used_idx];
 110                 desc_idx = (uint16_t) uep->id;
 111                 len[i] = uep->len;
 112                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
 113
 114                 if (unlikely(cookie == NULL)) {
 115                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
 116                                 vq->vq_used_cons_idx);
 117                         break;
 118                 }
 119
 120                 rte_prefetch0(cookie);
 121                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 122                 rx_pkts[i]  = cookie;
 123                 vq->vq_used_cons_idx++;
 124                 vq_ring_free_chain(vq, desc_idx);
 125                 vq->vq_descx[desc_idx].cookie = NULL;
 126         }
 127
 128         return i;
 129 }
 130
 131 static void
 132 virtqueue_dequeue_pkt_tx(struct virtqueue *vq)
 133 {
 134         struct vring_used_elem *uep;
 135         uint16_t used_idx, desc_idx;
 136
 137         used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 138         uep = &vq->vq_ring.used->ring[used_idx];
 139         desc_idx = (uint16_t) uep->id;
 140         vq->vq_used_cons_idx++;
 141         vq_ring_free_chain(vq, desc_idx);
 142 }
 143
 144
 145 static inline int
 146 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
 147 {
 148         struct vq_desc_extra *dxp;
 149         struct virtio_hw *hw = vq->hw;
 150         struct vring_desc *start_dp;
 151         uint16_t needed = 1;
 152         uint16_t head_idx, idx;
 153
 154         if (unlikely(vq->vq_free_cnt == 0))
 155                 return -ENOSPC;
 156         if (unlikely(vq->vq_free_cnt < needed))
 157                 return -EMSGSIZE;
 158
 159         head_idx = vq->vq_desc_head_idx;
 160         if (unlikely(head_idx >= vq->vq_nentries))
 161                 return -EFAULT;
 162
 163         idx = head_idx;
 164         dxp = &vq->vq_descx[idx];
 165         dxp->cookie = (void *)cookie;
 166         dxp->ndescs = needed;
 167
 168         start_dp = vq->vq_ring.desc;
 169         start_dp[idx].addr =
 170                 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM
 171                 - hw->vtnet_hdr_size);
 172         start_dp[idx].len =
 173                 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 174         start_dp[idx].flags =  VRING_DESC_F_WRITE;
 175         idx = start_dp[idx].next;
 176         vq->vq_desc_head_idx = idx;
 177         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 178                 vq->vq_desc_tail_idx = idx;
 179         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 180         vq_update_avail_ring(vq, head_idx);
 181
 182         return 0;
 183 }
 184
 185 static int
 186 virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
 187 {
 188         struct vq_desc_extra *dxp;
 189         struct vring_desc *start_dp;
 190         uint16_t seg_num = cookie->nb_segs;
 191         uint16_t needed = 1 + seg_num;
 192         uint16_t head_idx, idx;
 193         uint16_t head_size = txvq->hw->vtnet_hdr_size;
 194
 195         if (unlikely(txvq->vq_free_cnt == 0))
 196                 return -ENOSPC;
 197         if (unlikely(txvq->vq_free_cnt < needed))
 198                 return -EMSGSIZE;
 199         head_idx = txvq->vq_desc_head_idx;
 200         if (unlikely(head_idx >= txvq->vq_nentries))
 201                 return -EFAULT;
 202
 203         idx = head_idx;
 204         dxp = &txvq->vq_descx[idx];
 205         if (dxp->cookie != NULL)
 206                 rte_pktmbuf_free(dxp->cookie);
 207         dxp->cookie = (void *)cookie;
 208         dxp->ndescs = needed;
 209
 210         start_dp = txvq->vq_ring.desc;
 211         start_dp[idx].addr =
 212                 txvq->virtio_net_hdr_mem + idx * head_size;
 213         start_dp[idx].len = (uint32_t)head_size;
 214         start_dp[idx].flags = VRING_DESC_F_NEXT;
 215
 216         for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) {
 217                 idx = start_dp[idx].next;
 218                 start_dp[idx].addr  = RTE_MBUF_DATA_DMA_ADDR(cookie);
 219                 start_dp[idx].len   = cookie->data_len;
 220                 start_dp[idx].flags = VRING_DESC_F_NEXT;
 221                 cookie = cookie->next;
 222         }
 223
 224         start_dp[idx].flags &= ~VRING_DESC_F_NEXT;
 225         idx = start_dp[idx].next;
 226         txvq->vq_desc_head_idx = idx;
 227         if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 228                 txvq->vq_desc_tail_idx = idx;
 229         txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
 230         vq_update_avail_ring(txvq, head_idx);
 231
 232         return 0;
 233 }
 234
 235 static inline struct rte_mbuf *
 236 rte_rxmbuf_alloc(struct rte_mempool *mp)
 237 {
 238         struct rte_mbuf *m;
 239
 240         m = __rte_mbuf_raw_alloc(mp);
 241         __rte_mbuf_sanity_check_raw(m, 0);
 242
 243         return m;
 244 }
 245
 246 static void
 247 virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
 248 {
 249         struct rte_mbuf *m;
 250         int i, nbufs, error, size = vq->vq_nentries;
 251         struct vring *vr = &vq->vq_ring;
 252         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 253
 254         PMD_INIT_FUNC_TRACE();
 255
 256         /*
 257          * Reinitialise since virtio port might have been stopped and restarted
 258          */
 259         memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size);
 260         vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
 261         vq->vq_used_cons_idx = 0;
 262         vq->vq_desc_head_idx = 0;
 263         vq->vq_avail_idx = 0;
 264         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 265         vq->vq_free_cnt = vq->vq_nentries;
 266         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 267
 268         /* Chain all the descriptors in the ring with an END */
 269         for (i = 0; i < size - 1; i++)
 270                 vr->desc[i].next = (uint16_t)(i + 1);
 271         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 272
 273         /*
 274          * Disable device(host) interrupting guest
 275          */
 276         virtqueue_disable_intr(vq);
 277
 278         /* Only rx virtqueue needs mbufs to be allocated at initialization */
 279         if (queue_type == VTNET_RQ) {
 280                 if (vq->mpool == NULL)
 281                         rte_exit(EXIT_FAILURE,
 282                         "Cannot allocate initial mbufs for rx virtqueue");
 283
 284                 /* Allocate blank mbufs for the each rx descriptor */
 285                 nbufs = 0;
 286                 error = ENOSPC;
 287                 while (!virtqueue_full(vq)) {
 288                         m = rte_rxmbuf_alloc(vq->mpool);
 289                         if (m == NULL)
 290                                 break;
 291
 292                         /******************************************
 293                         *         Enqueue allocated buffers        *
 294                         *******************************************/
 295                         error = virtqueue_enqueue_recv_refill(vq, m);
 296
 297                         if (error) {
 298                                 rte_pktmbuf_free(m);
 299                                 break;
 300                         }
 301                         nbufs++;
 302                 }
 303
 304                 vq_update_avail_idx(vq);
 305
 306                 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
 307
 308                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 309                         vq->vq_queue_index);
 310                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 311                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 312         } else if (queue_type == VTNET_TQ) {
 313                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 314                         vq->vq_queue_index);
 315                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 316                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 317         } else {
 318                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 319                         vq->vq_queue_index);
 320                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 321                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 322         }
 323 }
 324
 325 void
 326 virtio_dev_cq_start(struct rte_eth_dev *dev)
 327 {
 328         struct virtio_hw *hw = dev->data->dev_private;
 329
 330         if (hw->cvq) {
 331                 virtio_dev_vring_start(hw->cvq, VTNET_CQ);
 332                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
 333         }
 334 }
 335
 336 void
 337 virtio_dev_rxtx_start(struct rte_eth_dev *dev)
 338 {
 339         /*
 340          * Start receive and transmit vrings
 341          * -    Setup vring structure for all queues
 342          * -    Initialize descriptor for the rx vring
 343          * -    Allocate blank mbufs for the each rx descriptor
 344          *
 345          */
 346         int i;
 347
 348         PMD_INIT_FUNC_TRACE();
 349
 350         /* Start rx vring. */
 351         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 352                 virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ);
 353                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 354         }
 355
 356         /* Start tx vring. */
 357         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 358                 virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ);
 359                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 360         }
 361 }
 362
 363 int
 364 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 365                         uint16_t queue_idx,
 366                         uint16_t nb_desc,
 367                         unsigned int socket_id,
 368                         __rte_unused const struct rte_eth_rxconf *rx_conf,
 369                         struct rte_mempool *mp)
 370 {
 371         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 372         struct virtqueue *vq;
 373         int ret;
 374
 375         PMD_INIT_FUNC_TRACE();
 376         ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
 377                         nb_desc, socket_id, &vq);
 378         if (ret < 0) {
 379                 PMD_INIT_LOG(ERR, "tvq initialization failed");
 380                 return ret;
 381         }
 382
 383         /* Create mempool for rx mbuf allocation */
 384         vq->mpool = mp;
 385
 386         dev->data->rx_queues[queue_idx] = vq;
 387         return 0;
 388 }
 389
 390 /*
 391  * struct rte_eth_dev *dev: Used to update dev
 392  * uint16_t nb_desc: Defaults to values read from config space
 393  * unsigned int socket_id: Used to allocate memzone
 394  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
 395  * uint16_t queue_idx: Just used as an index in dev txq list
 396  */
 397 int
 398 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 399                         uint16_t queue_idx,
 400                         uint16_t nb_desc,
 401                         unsigned int socket_id,
 402                         const struct rte_eth_txconf *tx_conf)
 403 {
 404         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 405         struct virtqueue *vq;
 406         int ret;
 407
 408         PMD_INIT_FUNC_TRACE();
 409
 410         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOOFFLOADS)
 411             != ETH_TXQ_FLAGS_NOOFFLOADS) {
 412                 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n");
 413                 return -EINVAL;
 414         }
 415
 416         ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
 417                         nb_desc, socket_id, &vq);
 418         if (ret < 0) {
 419                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 420                 return ret;
 421         }
 422
 423         dev->data->tx_queues[queue_idx] = vq;
 424         return 0;
 425 }
 426
 427 static void
 428 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 429 {
 430         int error;
 431         /*
 432          * Requeue the discarded mbuf. This should always be
 433          * successful since it was just dequeued.
 434          */
 435         error = virtqueue_enqueue_recv_refill(vq, m);
 436         if (unlikely(error)) {
 437                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
 438                 rte_pktmbuf_free(m);
 439         }
 440 }
 441
 442 #define VIRTIO_MBUF_BURST_SZ 64
 443 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 444 uint16_t
 445 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 446 {
 447         struct virtqueue *rxvq = rx_queue;
 448         struct rte_mbuf *rxm, *new_mbuf;
 449         uint16_t nb_used, num, nb_rx = 0;
 450         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 451         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 452         int error;
 453         uint32_t i, nb_enqueued = 0;
 454         const uint32_t hdr_size = sizeof(struct virtio_net_hdr);
 455
 456         nb_used = VIRTQUEUE_NUSED(rxvq);
 457
 458         virtio_rmb();
 459
 460         num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
 461         num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
 462         if (likely(num > DESC_PER_CACHELINE))
 463                 num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
 464
 465         if (num == 0)
 466                 return 0;
 467
 468         num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num);
 469         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
 470         for (i = 0; i < num ; i++) {
 471                 rxm = rcv_pkts[i];
 472
 473                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
 474
 475                 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
 476                         PMD_RX_LOG(ERR, "Packet drop");
 477                         nb_enqueued++;
 478                         virtio_discard_rxbuf(rxvq, rxm);
 479                         rxvq->errors++;
 480                         continue;
 481                 }
 482
 483                 rxm->port = rxvq->port_id;
 484                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 485
 486                 rxm->nb_segs = 1;
 487                 rxm->next = NULL;
 488                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
 489                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
 490
 491                 VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
 492
 493                 rx_pkts[nb_rx++] = rxm;
 494                 rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
 495         }
 496
 497         rxvq->packets += nb_rx;
 498
 499         /* Allocate new mbuf for the used descriptor */
 500         error = ENOSPC;
 501         while (likely(!virtqueue_full(rxvq))) {
 502                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 503                 if (unlikely(new_mbuf == NULL)) {
 504                         struct rte_eth_dev *dev
 505                                 = &rte_eth_devices[rxvq->port_id];
 506                         dev->data->rx_mbuf_alloc_failed++;
 507                         break;
 508                 }
 509                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 510                 if (unlikely(error)) {
 511                         rte_pktmbuf_free(new_mbuf);
 512                         break;
 513                 }
 514                 nb_enqueued++;
 515         }
 516
 517         if (likely(nb_enqueued)) {
 518                 vq_update_avail_idx(rxvq);
 519
 520                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 521                         virtqueue_notify(rxvq);
 522                         PMD_RX_LOG(DEBUG, "Notified\n");
 523                 }
 524         }
 525
 526         return nb_rx;
 527 }
 528
 529 uint16_t
 530 virtio_recv_mergeable_pkts(void *rx_queue,
 531                         struct rte_mbuf **rx_pkts,
 532                         uint16_t nb_pkts)
 533 {
 534         struct virtqueue *rxvq = rx_queue;
 535         struct rte_mbuf *rxm, *new_mbuf;
 536         uint16_t nb_used, num, nb_rx = 0;
 537         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 538         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 539         struct rte_mbuf *prev;
 540         int error;
 541         uint32_t i = 0, nb_enqueued = 0;
 542         uint32_t seg_num = 0;
 543         uint16_t extra_idx = 0;
 544         uint32_t seg_res = 0;
 545         const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 546
 547         nb_used = VIRTQUEUE_NUSED(rxvq);
 548
 549         virtio_rmb();
 550
 551         if (nb_used == 0)
 552                 return 0;
 553
 554         PMD_RX_LOG(DEBUG, "used:%d\n", nb_used);
 555
 556         while (i < nb_used) {
 557                 struct virtio_net_hdr_mrg_rxbuf *header;
 558
 559                 if (nb_rx == nb_pkts)
 560                         break;
 561
 562                 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1);
 563                 if (num != 1)
 564                         continue;
 565
 566                 i++;
 567
 568                 PMD_RX_LOG(DEBUG, "dequeue:%d\n", num);
 569                 PMD_RX_LOG(DEBUG, "packet len:%d\n", len[0]);
 570
 571                 rxm = rcv_pkts[0];
 572
 573                 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
 574                         PMD_RX_LOG(ERR, "Packet drop\n");
 575                         nb_enqueued++;
 576                         virtio_discard_rxbuf(rxvq, rxm);
 577                         rxvq->errors++;
 578                         continue;
 579                 }
 580
 581                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
 582                         RTE_PKTMBUF_HEADROOM - hdr_size);
 583                 seg_num = header->num_buffers;
 584
 585                 if (seg_num == 0)
 586                         seg_num = 1;
 587
 588                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 589                 rxm->nb_segs = seg_num;
 590                 rxm->next = NULL;
 591                 rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
 592                 rxm->data_len = (uint16_t)(len[0] - hdr_size);
 593
 594                 rxm->port = rxvq->port_id;
 595                 rx_pkts[nb_rx] = rxm;
 596                 prev = rxm;
 597
 598                 seg_res = seg_num - 1;
 599
 600                 while (seg_res != 0) {
 601                         /*
 602                          * Get extra segments for current uncompleted packet.
 603                          */
 604                         uint32_t  rcv_cnt =
 605                                 RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
 606                         if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) {
 607                                 uint32_t rx_num =
 608                                         virtqueue_dequeue_burst_rx(rxvq,
 609                                         rcv_pkts, len, rcv_cnt);
 610                                 i += rx_num;
 611                                 rcv_cnt = rx_num;
 612                         } else {
 613                                 PMD_RX_LOG(ERR,
 614                                         "No enough segments for packet.\n");
 615                                 nb_enqueued++;
 616                                 virtio_discard_rxbuf(rxvq, rxm);
 617                                 rxvq->errors++;
 618                                 break;
 619                         }
 620
 621                         extra_idx = 0;
 622
 623                         while (extra_idx < rcv_cnt) {
 624                                 rxm = rcv_pkts[extra_idx];
 625
 626                                 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
 627                                 rxm->next = NULL;
 628                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
 629                                 rxm->data_len = (uint16_t)(len[extra_idx]);
 630
 631                                 if (prev)
 632                                         prev->next = rxm;
 633
 634                                 prev = rxm;
 635                                 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len;
 636                                 extra_idx++;
 637                         };
 638                         seg_res -= rcv_cnt;
 639                 }
 640
 641                 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
 642                         rx_pkts[nb_rx]->data_len);
 643
 644                 rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
 645                 nb_rx++;
 646         }
 647
 648         rxvq->packets += nb_rx;
 649
 650         /* Allocate new mbuf for the used descriptor */
 651         error = ENOSPC;
 652         while (likely(!virtqueue_full(rxvq))) {
 653                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 654                 if (unlikely(new_mbuf == NULL)) {
 655                         struct rte_eth_dev *dev
 656                                 = &rte_eth_devices[rxvq->port_id];
 657                         dev->data->rx_mbuf_alloc_failed++;
 658                         break;
 659                 }
 660                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 661                 if (unlikely(error)) {
 662                         rte_pktmbuf_free(new_mbuf);
 663                         break;
 664                 }
 665                 nb_enqueued++;
 666         }
 667
 668         if (likely(nb_enqueued)) {
 669                 vq_update_avail_idx(rxvq);
 670
 671                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 672                         virtqueue_notify(rxvq);
 673                         PMD_RX_LOG(DEBUG, "Notified");
 674                 }
 675         }
 676
 677         return nb_rx;
 678 }
 679
 680 uint16_t
 681 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 682 {
 683         struct virtqueue *txvq = tx_queue;
 684         struct rte_mbuf *txm;
 685         uint16_t nb_used, nb_tx, num;
 686         int error;
 687
 688         nb_tx = 0;
 689
 690         if (unlikely(nb_pkts < 1))
 691                 return nb_pkts;
 692
 693         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 694         nb_used = VIRTQUEUE_NUSED(txvq);
 695
 696         virtio_rmb();
 697
 698         num = (uint16_t)(likely(nb_used < VIRTIO_MBUF_BURST_SZ) ? nb_used : VIRTIO_MBUF_BURST_SZ);
 699
 700         while (nb_tx < nb_pkts) {
 701                 /* Need one more descriptor for virtio header. */
 702                 int need = tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1;
 703                 int deq_cnt = RTE_MIN(need, (int)num);
 704
 705                 num -= (deq_cnt > 0) ? deq_cnt : 0;
 706                 while (deq_cnt > 0) {
 707                         virtqueue_dequeue_pkt_tx(txvq);
 708                         deq_cnt--;
 709                 }
 710
 711                 need = (int)tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1;
 712                 /*
 713                  * Zero or negative value indicates it has enough free
 714                  * descriptors to use for transmitting.
 715                  */
 716                 if (likely(need <= 0)) {
 717                         txm = tx_pkts[nb_tx];
 718                         /* Enqueue Packet buffers */
 719                         error = virtqueue_enqueue_xmit(txvq, txm);
 720                         if (unlikely(error)) {
 721                                 if (error == ENOSPC)
 722                                         PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0");
 723                                 else if (error == EMSGSIZE)
 724                                         PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1");
 725                                 else
 726                                         PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error);
 727                                 break;
 728                         }
 729                         nb_tx++;
 730                         txvq->bytes += txm->pkt_len;
 731                 } else {
 732                         PMD_TX_LOG(ERR, "No free tx descriptors to transmit");
 733                         break;
 734                 }
 735         }
 736
 737         txvq->packets += nb_tx;
 738
 739         if (likely(nb_tx)) {
 740                 vq_update_avail_idx(txvq);
 741
 742                 if (unlikely(virtqueue_kick_prepare(txvq))) {
 743                         virtqueue_notify(txvq);
 744                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
 745                 }
 746         }
 747
 748         return nb_tx;
 749 }