drivers/net/virtio/virtio_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 #include <rte_cycles.h>
  41 #include <rte_memory.h>
  42 #include <rte_memzone.h>
  43 #include <rte_branch_prediction.h>
  44 #include <rte_mempool.h>
  45 #include <rte_malloc.h>
  46 #include <rte_mbuf.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_prefetch.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_errno.h>
  52 #include <rte_byteorder.h>
  53
  54 #include "virtio_logs.h"
  55 #include "virtio_ethdev.h"
  56 #include "virtio_pci.h"
  57 #include "virtqueue.h"
  58 #include "virtio_rxtx.h"
  59
  60 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
  61 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
  62 #else
  63 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
  64 #endif
  65
  66
  67 #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
  68         ETH_TXQ_FLAGS_NOOFFLOADS)
  69
  70 static int use_simple_rxtx;
  71
  72 static void
  73 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  74 {
  75         struct vring_desc *dp, *dp_tail;
  76         struct vq_desc_extra *dxp;
  77         uint16_t desc_idx_last = desc_idx;
  78
  79         dp  = &vq->vq_ring.desc[desc_idx];
  80         dxp = &vq->vq_descx[desc_idx];
  81         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
  82         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
  83                 while (dp->flags & VRING_DESC_F_NEXT) {
  84                         desc_idx_last = dp->next;
  85                         dp = &vq->vq_ring.desc[dp->next];
  86                 }
  87         }
  88         dxp->ndescs = 0;
  89
  90         /*
  91          * We must append the existing free chain, if any, to the end of
  92          * newly freed chain. If the virtqueue was completely used, then
  93          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  94          */
  95         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
  96                 vq->vq_desc_head_idx = desc_idx;
  97         } else {
  98                 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
  99                 dp_tail->next = desc_idx;
 100         }
 101
 102         vq->vq_desc_tail_idx = desc_idx_last;
 103         dp->next = VQ_RING_DESC_CHAIN_END;
 104 }
 105
 106 static uint16_t
 107 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 108                            uint32_t *len, uint16_t num)
 109 {
 110         struct vring_used_elem *uep;
 111         struct rte_mbuf *cookie;
 112         uint16_t used_idx, desc_idx;
 113         uint16_t i;
 114
 115         /*  Caller does the check */
 116         for (i = 0; i < num ; i++) {
 117                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 118                 uep = &vq->vq_ring.used->ring[used_idx];
 119                 desc_idx = (uint16_t) uep->id;
 120                 len[i] = uep->len;
 121                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
 122
 123                 if (unlikely(cookie == NULL)) {
 124                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
 125                                 vq->vq_used_cons_idx);
 126                         break;
 127                 }
 128
 129                 rte_prefetch0(cookie);
 130                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 131                 rx_pkts[i]  = cookie;
 132                 vq->vq_used_cons_idx++;
 133                 vq_ring_free_chain(vq, desc_idx);
 134                 vq->vq_descx[desc_idx].cookie = NULL;
 135         }
 136
 137         return i;
 138 }
 139
 140 #ifndef DEFAULT_TX_FREE_THRESH
 141 #define DEFAULT_TX_FREE_THRESH 32
 142 #endif
 143
 144 /* Cleanup from completed transmits. */
 145 static void
 146 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
 147 {
 148         uint16_t i, used_idx, desc_idx;
 149         for (i = 0; i < num; i++) {
 150                 struct vring_used_elem *uep;
 151                 struct vq_desc_extra *dxp;
 152
 153                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 154                 uep = &vq->vq_ring.used->ring[used_idx];
 155
 156                 desc_idx = (uint16_t) uep->id;
 157                 dxp = &vq->vq_descx[desc_idx];
 158                 vq->vq_used_cons_idx++;
 159                 vq_ring_free_chain(vq, desc_idx);
 160
 161                 if (dxp->cookie != NULL) {
 162                         rte_pktmbuf_free(dxp->cookie);
 163                         dxp->cookie = NULL;
 164                 }
 165         }
 166 }
 167
 168
 169 static inline int
 170 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
 171 {
 172         struct vq_desc_extra *dxp;
 173         struct virtio_hw *hw = vq->hw;
 174         struct vring_desc *start_dp;
 175         uint16_t needed = 1;
 176         uint16_t head_idx, idx;
 177
 178         if (unlikely(vq->vq_free_cnt == 0))
 179                 return -ENOSPC;
 180         if (unlikely(vq->vq_free_cnt < needed))
 181                 return -EMSGSIZE;
 182
 183         head_idx = vq->vq_desc_head_idx;
 184         if (unlikely(head_idx >= vq->vq_nentries))
 185                 return -EFAULT;
 186
 187         idx = head_idx;
 188         dxp = &vq->vq_descx[idx];
 189         dxp->cookie = (void *)cookie;
 190         dxp->ndescs = needed;
 191
 192         start_dp = vq->vq_ring.desc;
 193         start_dp[idx].addr =
 194                 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM
 195                 - hw->vtnet_hdr_size);
 196         start_dp[idx].len =
 197                 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 198         start_dp[idx].flags =  VRING_DESC_F_WRITE;
 199         idx = start_dp[idx].next;
 200         vq->vq_desc_head_idx = idx;
 201         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 202                 vq->vq_desc_tail_idx = idx;
 203         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 204         vq_update_avail_ring(vq, head_idx);
 205
 206         return 0;
 207 }
 208
 209 static int
 210 virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
 211 {
 212         struct vq_desc_extra *dxp;
 213         struct vring_desc *start_dp;
 214         uint16_t seg_num = cookie->nb_segs;
 215         uint16_t needed = 1 + seg_num;
 216         uint16_t head_idx, idx;
 217         size_t head_size = txvq->hw->vtnet_hdr_size;
 218
 219         if (unlikely(txvq->vq_free_cnt == 0))
 220                 return -ENOSPC;
 221         if (unlikely(txvq->vq_free_cnt < needed))
 222                 return -EMSGSIZE;
 223         head_idx = txvq->vq_desc_head_idx;
 224         if (unlikely(head_idx >= txvq->vq_nentries))
 225                 return -EFAULT;
 226
 227         idx = head_idx;
 228         dxp = &txvq->vq_descx[idx];
 229         dxp->cookie = (void *)cookie;
 230         dxp->ndescs = needed;
 231
 232         start_dp = txvq->vq_ring.desc;
 233         start_dp[idx].addr =
 234                 txvq->virtio_net_hdr_mem + idx * head_size;
 235         start_dp[idx].len = head_size;
 236         start_dp[idx].flags = VRING_DESC_F_NEXT;
 237
 238         for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) {
 239                 idx = start_dp[idx].next;
 240                 start_dp[idx].addr  = RTE_MBUF_DATA_DMA_ADDR(cookie);
 241                 start_dp[idx].len   = cookie->data_len;
 242                 start_dp[idx].flags = VRING_DESC_F_NEXT;
 243                 cookie = cookie->next;
 244         }
 245
 246         start_dp[idx].flags &= ~VRING_DESC_F_NEXT;
 247         idx = start_dp[idx].next;
 248         txvq->vq_desc_head_idx = idx;
 249         if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 250                 txvq->vq_desc_tail_idx = idx;
 251         txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
 252         vq_update_avail_ring(txvq, head_idx);
 253
 254         return 0;
 255 }
 256
 257 static inline struct rte_mbuf *
 258 rte_rxmbuf_alloc(struct rte_mempool *mp)
 259 {
 260         struct rte_mbuf *m;
 261
 262         m = __rte_mbuf_raw_alloc(mp);
 263         __rte_mbuf_sanity_check_raw(m, 0);
 264
 265         return m;
 266 }
 267
 268 static void
 269 virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
 270 {
 271         struct rte_mbuf *m;
 272         int i, nbufs, error, size = vq->vq_nentries;
 273         struct vring *vr = &vq->vq_ring;
 274         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 275
 276         PMD_INIT_FUNC_TRACE();
 277
 278         /*
 279          * Reinitialise since virtio port might have been stopped and restarted
 280          */
 281         memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size);
 282         vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
 283         vq->vq_used_cons_idx = 0;
 284         vq->vq_desc_head_idx = 0;
 285         vq->vq_avail_idx = 0;
 286         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 287         vq->vq_free_cnt = vq->vq_nentries;
 288         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 289
 290         /* Chain all the descriptors in the ring with an END */
 291         for (i = 0; i < size - 1; i++)
 292                 vr->desc[i].next = (uint16_t)(i + 1);
 293         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 294
 295         /*
 296          * Disable device(host) interrupting guest
 297          */
 298         virtqueue_disable_intr(vq);
 299
 300         /* Only rx virtqueue needs mbufs to be allocated at initialization */
 301         if (queue_type == VTNET_RQ) {
 302                 if (vq->mpool == NULL)
 303                         rte_exit(EXIT_FAILURE,
 304                         "Cannot allocate initial mbufs for rx virtqueue");
 305
 306                 /* Allocate blank mbufs for the each rx descriptor */
 307                 nbufs = 0;
 308                 error = ENOSPC;
 309
 310                 if (use_simple_rxtx)
 311                         for (i = 0; i < vq->vq_nentries; i++) {
 312                                 vq->vq_ring.avail->ring[i] = i;
 313                                 vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
 314                         }
 315
 316                 memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf));
 317                 for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++)
 318                         vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf;
 319
 320                 while (!virtqueue_full(vq)) {
 321                         m = rte_rxmbuf_alloc(vq->mpool);
 322                         if (m == NULL)
 323                                 break;
 324
 325                         /******************************************
 326                         *         Enqueue allocated buffers        *
 327                         *******************************************/
 328                         if (use_simple_rxtx)
 329                                 error = virtqueue_enqueue_recv_refill_simple(vq, m);
 330                         else
 331                                 error = virtqueue_enqueue_recv_refill(vq, m);
 332                         if (error) {
 333                                 rte_pktmbuf_free(m);
 334                                 break;
 335                         }
 336                         nbufs++;
 337                 }
 338
 339                 vq_update_avail_idx(vq);
 340
 341                 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
 342         } else if (queue_type == VTNET_TQ) {
 343                 if (use_simple_rxtx) {
 344                         int mid_idx  = vq->vq_nentries >> 1;
 345                         for (i = 0; i < mid_idx; i++) {
 346                                 vq->vq_ring.avail->ring[i] = i + mid_idx;
 347                                 vq->vq_ring.desc[i + mid_idx].next = i;
 348                                 vq->vq_ring.desc[i + mid_idx].addr =
 349                                         vq->virtio_net_hdr_mem +
 350                                                 mid_idx * vq->hw->vtnet_hdr_size;
 351                                 vq->vq_ring.desc[i + mid_idx].len =
 352                                         vq->hw->vtnet_hdr_size;
 353                                 vq->vq_ring.desc[i + mid_idx].flags =
 354                                         VRING_DESC_F_NEXT;
 355                                 vq->vq_ring.desc[i].flags = 0;
 356                         }
 357                         for (i = mid_idx; i < vq->vq_nentries; i++)
 358                                 vq->vq_ring.avail->ring[i] = i;
 359                 }
 360         }
 361 }
 362
 363 void
 364 virtio_dev_cq_start(struct rte_eth_dev *dev)
 365 {
 366         struct virtio_hw *hw = dev->data->dev_private;
 367
 368         if (hw->cvq) {
 369                 virtio_dev_vring_start(hw->cvq, VTNET_CQ);
 370                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
 371         }
 372 }
 373
 374 void
 375 virtio_dev_rxtx_start(struct rte_eth_dev *dev)
 376 {
 377         /*
 378          * Start receive and transmit vrings
 379          * -    Setup vring structure for all queues
 380          * -    Initialize descriptor for the rx vring
 381          * -    Allocate blank mbufs for the each rx descriptor
 382          *
 383          */
 384         int i;
 385
 386         PMD_INIT_FUNC_TRACE();
 387
 388         /* Start rx vring. */
 389         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 390                 virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ);
 391                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 392         }
 393
 394         /* Start tx vring. */
 395         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 396                 virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ);
 397                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 398         }
 399 }
 400
 401 int
 402 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 403                         uint16_t queue_idx,
 404                         uint16_t nb_desc,
 405                         unsigned int socket_id,
 406                         __rte_unused const struct rte_eth_rxconf *rx_conf,
 407                         struct rte_mempool *mp)
 408 {
 409         uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 410         struct virtqueue *vq;
 411         int ret;
 412
 413         PMD_INIT_FUNC_TRACE();
 414         ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
 415                         nb_desc, socket_id, &vq);
 416         if (ret < 0) {
 417                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 418                 return ret;
 419         }
 420
 421         /* Create mempool for rx mbuf allocation */
 422         vq->mpool = mp;
 423
 424         dev->data->rx_queues[queue_idx] = vq;
 425
 426         virtio_rxq_vec_setup(vq);
 427
 428         return 0;
 429 }
 430
 431 void
 432 virtio_dev_rx_queue_release(void *rxq)
 433 {
 434         virtio_dev_queue_release(rxq);
 435 }
 436
 437 /*
 438  * struct rte_eth_dev *dev: Used to update dev
 439  * uint16_t nb_desc: Defaults to values read from config space
 440  * unsigned int socket_id: Used to allocate memzone
 441  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
 442  * uint16_t queue_idx: Just used as an index in dev txq list
 443  */
 444 int
 445 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 446                         uint16_t queue_idx,
 447                         uint16_t nb_desc,
 448                         unsigned int socket_id,
 449                         const struct rte_eth_txconf *tx_conf)
 450 {
 451         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 452         struct virtio_hw *hw = dev->data->dev_private;
 453         struct virtqueue *vq;
 454         uint16_t tx_free_thresh;
 455         int ret;
 456
 457         PMD_INIT_FUNC_TRACE();
 458
 459         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS)
 460             != ETH_TXQ_FLAGS_NOXSUMS) {
 461                 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n");
 462                 return -EINVAL;
 463         }
 464
 465         /* Use simple rx/tx func if single segment and no offloads */
 466         if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS &&
 467              !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
 468                 PMD_INIT_LOG(INFO, "Using simple rx/tx path");
 469                 dev->tx_pkt_burst = virtio_xmit_pkts_simple;
 470                 dev->rx_pkt_burst = virtio_recv_pkts_vec;
 471                 use_simple_rxtx = 1;
 472         }
 473
 474         ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
 475                         nb_desc, socket_id, &vq);
 476         if (ret < 0) {
 477                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 478                 return ret;
 479         }
 480
 481         tx_free_thresh = tx_conf->tx_free_thresh;
 482         if (tx_free_thresh == 0)
 483                 tx_free_thresh =
 484                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
 485
 486         if (tx_free_thresh >= (vq->vq_nentries - 3)) {
 487                 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the "
 488                         "number of TX entries minus 3 (%u)."
 489                         " (tx_free_thresh=%u port=%u queue=%u)\n",
 490                         vq->vq_nentries - 3,
 491                         tx_free_thresh, dev->data->port_id, queue_idx);
 492                 return -EINVAL;
 493         }
 494
 495         vq->vq_free_thresh = tx_free_thresh;
 496
 497         dev->data->tx_queues[queue_idx] = vq;
 498         return 0;
 499 }
 500
 501 void
 502 virtio_dev_tx_queue_release(void *txq)
 503 {
 504         virtio_dev_queue_release(txq);
 505 }
 506
 507 static void
 508 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 509 {
 510         int error;
 511         /*
 512          * Requeue the discarded mbuf. This should always be
 513          * successful since it was just dequeued.
 514          */
 515         error = virtqueue_enqueue_recv_refill(vq, m);
 516         if (unlikely(error)) {
 517                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
 518                 rte_pktmbuf_free(m);
 519         }
 520 }
 521
 522 static void
 523 virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf)
 524 {
 525         uint32_t s = mbuf->pkt_len;
 526         struct ether_addr *ea;
 527
 528         if (s == 64) {
 529                 vq->size_bins[1]++;
 530         } else if (s > 64 && s < 1024) {
 531                 uint32_t bin;
 532
 533                 /* count zeros, and offset into correct bin */
 534                 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
 535                 vq->size_bins[bin]++;
 536         } else {
 537                 if (s < 64)
 538                         vq->size_bins[0]++;
 539                 else if (s < 1519)
 540                         vq->size_bins[6]++;
 541                 else if (s >= 1519)
 542                         vq->size_bins[7]++;
 543         }
 544
 545         ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
 546         vq->multicast += is_multicast_ether_addr(ea);
 547         vq->broadcast += is_broadcast_ether_addr(ea);
 548 }
 549
 550 #define VIRTIO_MBUF_BURST_SZ 64
 551 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 552 uint16_t
 553 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 554 {
 555         struct virtqueue *rxvq = rx_queue;
 556         struct virtio_hw *hw;
 557         struct rte_mbuf *rxm, *new_mbuf;
 558         uint16_t nb_used, num, nb_rx;
 559         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 560         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 561         int error;
 562         uint32_t i, nb_enqueued;
 563         const uint32_t hdr_size = sizeof(struct virtio_net_hdr);
 564
 565         nb_used = VIRTQUEUE_NUSED(rxvq);
 566
 567         virtio_rmb();
 568
 569         num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
 570         num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
 571         if (likely(num > DESC_PER_CACHELINE))
 572                 num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
 573
 574         if (num == 0)
 575                 return 0;
 576
 577         num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num);
 578         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
 579
 580         hw = rxvq->hw;
 581         nb_rx = 0;
 582         nb_enqueued = 0;
 583
 584         for (i = 0; i < num ; i++) {
 585                 rxm = rcv_pkts[i];
 586
 587                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
 588
 589                 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
 590                         PMD_RX_LOG(ERR, "Packet drop");
 591                         nb_enqueued++;
 592                         virtio_discard_rxbuf(rxvq, rxm);
 593                         rxvq->errors++;
 594                         continue;
 595                 }
 596
 597                 rxm->port = rxvq->port_id;
 598                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 599                 rxm->ol_flags = 0;
 600                 rxm->vlan_tci = 0;
 601
 602                 rxm->nb_segs = 1;
 603                 rxm->next = NULL;
 604                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
 605                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
 606
 607                 if (hw->vlan_strip)
 608                         rte_vlan_strip(rxm);
 609
 610                 VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
 611
 612                 rx_pkts[nb_rx++] = rxm;
 613
 614                 rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
 615                 virtio_update_packet_stats(rxvq, rxm);
 616         }
 617
 618         rxvq->packets += nb_rx;
 619
 620         /* Allocate new mbuf for the used descriptor */
 621         error = ENOSPC;
 622         while (likely(!virtqueue_full(rxvq))) {
 623                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 624                 if (unlikely(new_mbuf == NULL)) {
 625                         struct rte_eth_dev *dev
 626                                 = &rte_eth_devices[rxvq->port_id];
 627                         dev->data->rx_mbuf_alloc_failed++;
 628                         break;
 629                 }
 630                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 631                 if (unlikely(error)) {
 632                         rte_pktmbuf_free(new_mbuf);
 633                         break;
 634                 }
 635                 nb_enqueued++;
 636         }
 637
 638         if (likely(nb_enqueued)) {
 639                 vq_update_avail_idx(rxvq);
 640
 641                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 642                         virtqueue_notify(rxvq);
 643                         PMD_RX_LOG(DEBUG, "Notified\n");
 644                 }
 645         }
 646
 647         return nb_rx;
 648 }
 649
 650 uint16_t
 651 virtio_recv_mergeable_pkts(void *rx_queue,
 652                         struct rte_mbuf **rx_pkts,
 653                         uint16_t nb_pkts)
 654 {
 655         struct virtqueue *rxvq = rx_queue;
 656         struct virtio_hw *hw;
 657         struct rte_mbuf *rxm, *new_mbuf;
 658         uint16_t nb_used, num, nb_rx;
 659         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 660         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 661         struct rte_mbuf *prev;
 662         int error;
 663         uint32_t i, nb_enqueued;
 664         uint32_t seg_num;
 665         uint16_t extra_idx;
 666         uint32_t seg_res;
 667         const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 668
 669         nb_used = VIRTQUEUE_NUSED(rxvq);
 670
 671         virtio_rmb();
 672
 673         if (nb_used == 0)
 674                 return 0;
 675
 676         PMD_RX_LOG(DEBUG, "used:%d\n", nb_used);
 677
 678         hw = rxvq->hw;
 679         nb_rx = 0;
 680         i = 0;
 681         nb_enqueued = 0;
 682         seg_num = 0;
 683         extra_idx = 0;
 684         seg_res = 0;
 685
 686         while (i < nb_used) {
 687                 struct virtio_net_hdr_mrg_rxbuf *header;
 688
 689                 if (nb_rx == nb_pkts)
 690                         break;
 691
 692                 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1);
 693                 if (num != 1)
 694                         continue;
 695
 696                 i++;
 697
 698                 PMD_RX_LOG(DEBUG, "dequeue:%d\n", num);
 699                 PMD_RX_LOG(DEBUG, "packet len:%d\n", len[0]);
 700
 701                 rxm = rcv_pkts[0];
 702
 703                 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
 704                         PMD_RX_LOG(ERR, "Packet drop\n");
 705                         nb_enqueued++;
 706                         virtio_discard_rxbuf(rxvq, rxm);
 707                         rxvq->errors++;
 708                         continue;
 709                 }
 710
 711                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
 712                         RTE_PKTMBUF_HEADROOM - hdr_size);
 713                 seg_num = header->num_buffers;
 714
 715                 if (seg_num == 0)
 716                         seg_num = 1;
 717
 718                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 719                 rxm->nb_segs = seg_num;
 720                 rxm->next = NULL;
 721                 rxm->ol_flags = 0;
 722                 rxm->vlan_tci = 0;
 723                 rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
 724                 rxm->data_len = (uint16_t)(len[0] - hdr_size);
 725
 726                 rxm->port = rxvq->port_id;
 727                 rx_pkts[nb_rx] = rxm;
 728                 prev = rxm;
 729
 730                 seg_res = seg_num - 1;
 731
 732                 while (seg_res != 0) {
 733                         /*
 734                          * Get extra segments for current uncompleted packet.
 735                          */
 736                         uint16_t  rcv_cnt =
 737                                 RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
 738                         if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) {
 739                                 uint32_t rx_num =
 740                                         virtqueue_dequeue_burst_rx(rxvq,
 741                                         rcv_pkts, len, rcv_cnt);
 742                                 i += rx_num;
 743                                 rcv_cnt = rx_num;
 744                         } else {
 745                                 PMD_RX_LOG(ERR,
 746                                         "No enough segments for packet.\n");
 747                                 nb_enqueued++;
 748                                 virtio_discard_rxbuf(rxvq, rxm);
 749                                 rxvq->errors++;
 750                                 break;
 751                         }
 752
 753                         extra_idx = 0;
 754
 755                         while (extra_idx < rcv_cnt) {
 756                                 rxm = rcv_pkts[extra_idx];
 757
 758                                 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
 759                                 rxm->next = NULL;
 760                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
 761                                 rxm->data_len = (uint16_t)(len[extra_idx]);
 762
 763                                 if (prev)
 764                                         prev->next = rxm;
 765
 766                                 prev = rxm;
 767                                 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len;
 768                                 extra_idx++;
 769                         };
 770                         seg_res -= rcv_cnt;
 771                 }
 772
 773                 if (hw->vlan_strip)
 774                         rte_vlan_strip(rx_pkts[nb_rx]);
 775
 776                 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
 777                         rx_pkts[nb_rx]->data_len);
 778
 779                 rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
 780                 virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]);
 781                 nb_rx++;
 782         }
 783
 784         rxvq->packets += nb_rx;
 785
 786         /* Allocate new mbuf for the used descriptor */
 787         error = ENOSPC;
 788         while (likely(!virtqueue_full(rxvq))) {
 789                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 790                 if (unlikely(new_mbuf == NULL)) {
 791                         struct rte_eth_dev *dev
 792                                 = &rte_eth_devices[rxvq->port_id];
 793                         dev->data->rx_mbuf_alloc_failed++;
 794                         break;
 795                 }
 796                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 797                 if (unlikely(error)) {
 798                         rte_pktmbuf_free(new_mbuf);
 799                         break;
 800                 }
 801                 nb_enqueued++;
 802         }
 803
 804         if (likely(nb_enqueued)) {
 805                 vq_update_avail_idx(rxvq);
 806
 807                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 808                         virtqueue_notify(rxvq);
 809                         PMD_RX_LOG(DEBUG, "Notified");
 810                 }
 811         }
 812
 813         return nb_rx;
 814 }
 815
 816 uint16_t
 817 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 818 {
 819         struct virtqueue *txvq = tx_queue;
 820         uint16_t nb_used, nb_tx;
 821         int error;
 822
 823         if (unlikely(nb_pkts < 1))
 824                 return nb_pkts;
 825
 826         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 827         nb_used = VIRTQUEUE_NUSED(txvq);
 828
 829         virtio_rmb();
 830         if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh))
 831                 virtio_xmit_cleanup(txvq, nb_used);
 832
 833         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
 834                 struct rte_mbuf *txm = tx_pkts[nb_tx];
 835                 /* Need one more descriptor for virtio header. */
 836                 int need = txm->nb_segs - txvq->vq_free_cnt + 1;
 837
 838                 /* Positive value indicates it need free vring descriptors */
 839                 if (unlikely(need > 0)) {
 840                         nb_used = VIRTQUEUE_NUSED(txvq);
 841                         virtio_rmb();
 842                         need = RTE_MIN(need, (int)nb_used);
 843
 844                         virtio_xmit_cleanup(txvq, need);
 845                         need = txm->nb_segs - txvq->vq_free_cnt + 1;
 846                         if (unlikely(need > 0)) {
 847                                 PMD_TX_LOG(ERR,
 848                                            "No free tx descriptors to transmit");
 849                                 break;
 850                         }
 851                 }
 852
 853                 /* Do VLAN tag insertion */
 854                 if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
 855                         error = rte_vlan_insert(&txm);
 856                         if (unlikely(error)) {
 857                                 rte_pktmbuf_free(txm);
 858                                 continue;
 859                         }
 860                 }
 861
 862                 /* Enqueue Packet buffers */
 863                 error = virtqueue_enqueue_xmit(txvq, txm);
 864                 if (unlikely(error)) {
 865                         if (error == ENOSPC)
 866                                 PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0");
 867                         else if (error == EMSGSIZE)
 868                                 PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1");
 869                         else
 870                                 PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error);
 871                         break;
 872                 }
 873
 874                 txvq->bytes += txm->pkt_len;
 875                 virtio_update_packet_stats(txvq, txm);
 876         }
 877
 878         txvq->packets += nb_tx;
 879
 880         if (likely(nb_tx)) {
 881                 vq_update_avail_idx(txvq);
 882
 883                 if (unlikely(virtqueue_kick_prepare(txvq))) {
 884                         virtqueue_notify(txvq);
 885                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
 886                 }
 887         }
 888
 889         return nb_tx;
 890 }