drivers/net/virtio/virtio_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 #include <rte_cycles.h>
  41 #include <rte_memory.h>
  42 #include <rte_memzone.h>
  43 #include <rte_branch_prediction.h>
  44 #include <rte_mempool.h>
  45 #include <rte_malloc.h>
  46 #include <rte_mbuf.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_prefetch.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_errno.h>
  52 #include <rte_byteorder.h>
  53
  54 #include "virtio_logs.h"
  55 #include "virtio_ethdev.h"
  56 #include "virtqueue.h"
  57 #include "virtio_rxtx.h"
  58
  59 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
  60 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
  61 #else
  62 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
  63 #endif
  64
  65 static int use_simple_rxtx;
  66
  67 static void
  68 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  69 {
  70         struct vring_desc *dp, *dp_tail;
  71         struct vq_desc_extra *dxp;
  72         uint16_t desc_idx_last = desc_idx;
  73
  74         dp  = &vq->vq_ring.desc[desc_idx];
  75         dxp = &vq->vq_descx[desc_idx];
  76         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
  77         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
  78                 while (dp->flags & VRING_DESC_F_NEXT) {
  79                         desc_idx_last = dp->next;
  80                         dp = &vq->vq_ring.desc[dp->next];
  81                 }
  82         }
  83         dxp->ndescs = 0;
  84
  85         /*
  86          * We must append the existing free chain, if any, to the end of
  87          * newly freed chain. If the virtqueue was completely used, then
  88          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  89          */
  90         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
  91                 vq->vq_desc_head_idx = desc_idx;
  92         } else {
  93                 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
  94                 dp_tail->next = desc_idx;
  95         }
  96
  97         vq->vq_desc_tail_idx = desc_idx_last;
  98         dp->next = VQ_RING_DESC_CHAIN_END;
  99 }
 100
 101 static uint16_t
 102 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 103                            uint32_t *len, uint16_t num)
 104 {
 105         struct vring_used_elem *uep;
 106         struct rte_mbuf *cookie;
 107         uint16_t used_idx, desc_idx;
 108         uint16_t i;
 109
 110         /*  Caller does the check */
 111         for (i = 0; i < num ; i++) {
 112                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 113                 uep = &vq->vq_ring.used->ring[used_idx];
 114                 desc_idx = (uint16_t) uep->id;
 115                 len[i] = uep->len;
 116                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
 117
 118                 if (unlikely(cookie == NULL)) {
 119                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
 120                                 vq->vq_used_cons_idx);
 121                         break;
 122                 }
 123
 124                 rte_prefetch0(cookie);
 125                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 126                 rx_pkts[i]  = cookie;
 127                 vq->vq_used_cons_idx++;
 128                 vq_ring_free_chain(vq, desc_idx);
 129                 vq->vq_descx[desc_idx].cookie = NULL;
 130         }
 131
 132         return i;
 133 }
 134
 135 #ifndef DEFAULT_TX_FREE_THRESH
 136 #define DEFAULT_TX_FREE_THRESH 32
 137 #endif
 138
 139 /* Cleanup from completed transmits. */
 140 static void
 141 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
 142 {
 143         uint16_t i, used_idx, desc_idx;
 144         for (i = 0; i < num; i++) {
 145                 struct vring_used_elem *uep;
 146                 struct vq_desc_extra *dxp;
 147
 148                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 149                 uep = &vq->vq_ring.used->ring[used_idx];
 150
 151                 desc_idx = (uint16_t) uep->id;
 152                 dxp = &vq->vq_descx[desc_idx];
 153                 vq->vq_used_cons_idx++;
 154                 vq_ring_free_chain(vq, desc_idx);
 155
 156                 if (dxp->cookie != NULL) {
 157                         rte_pktmbuf_free(dxp->cookie);
 158                         dxp->cookie = NULL;
 159                 }
 160         }
 161 }
 162
 163
 164 static inline int
 165 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
 166 {
 167         struct vq_desc_extra *dxp;
 168         struct virtio_hw *hw = vq->hw;
 169         struct vring_desc *start_dp;
 170         uint16_t needed = 1;
 171         uint16_t head_idx, idx;
 172
 173         if (unlikely(vq->vq_free_cnt == 0))
 174                 return -ENOSPC;
 175         if (unlikely(vq->vq_free_cnt < needed))
 176                 return -EMSGSIZE;
 177
 178         head_idx = vq->vq_desc_head_idx;
 179         if (unlikely(head_idx >= vq->vq_nentries))
 180                 return -EFAULT;
 181
 182         idx = head_idx;
 183         dxp = &vq->vq_descx[idx];
 184         dxp->cookie = (void *)cookie;
 185         dxp->ndescs = needed;
 186
 187         start_dp = vq->vq_ring.desc;
 188         start_dp[idx].addr =
 189                 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM
 190                 - hw->vtnet_hdr_size);
 191         start_dp[idx].len =
 192                 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 193         start_dp[idx].flags =  VRING_DESC_F_WRITE;
 194         idx = start_dp[idx].next;
 195         vq->vq_desc_head_idx = idx;
 196         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 197                 vq->vq_desc_tail_idx = idx;
 198         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 199         vq_update_avail_ring(vq, head_idx);
 200
 201         return 0;
 202 }
 203
 204 static int
 205 virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
 206 {
 207         struct vq_desc_extra *dxp;
 208         struct vring_desc *start_dp;
 209         uint16_t seg_num = cookie->nb_segs;
 210         uint16_t needed = 1 + seg_num;
 211         uint16_t head_idx, idx;
 212         size_t head_size = txvq->hw->vtnet_hdr_size;
 213
 214         if (unlikely(txvq->vq_free_cnt == 0))
 215                 return -ENOSPC;
 216         if (unlikely(txvq->vq_free_cnt < needed))
 217                 return -EMSGSIZE;
 218         head_idx = txvq->vq_desc_head_idx;
 219         if (unlikely(head_idx >= txvq->vq_nentries))
 220                 return -EFAULT;
 221
 222         idx = head_idx;
 223         dxp = &txvq->vq_descx[idx];
 224         dxp->cookie = (void *)cookie;
 225         dxp->ndescs = needed;
 226
 227         start_dp = txvq->vq_ring.desc;
 228         start_dp[idx].addr =
 229                 txvq->virtio_net_hdr_mem + idx * head_size;
 230         start_dp[idx].len = head_size;
 231         start_dp[idx].flags = VRING_DESC_F_NEXT;
 232
 233         for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) {
 234                 idx = start_dp[idx].next;
 235                 start_dp[idx].addr  = RTE_MBUF_DATA_DMA_ADDR(cookie);
 236                 start_dp[idx].len   = cookie->data_len;
 237                 start_dp[idx].flags = VRING_DESC_F_NEXT;
 238                 cookie = cookie->next;
 239         }
 240
 241         start_dp[idx].flags &= ~VRING_DESC_F_NEXT;
 242         idx = start_dp[idx].next;
 243         txvq->vq_desc_head_idx = idx;
 244         if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 245                 txvq->vq_desc_tail_idx = idx;
 246         txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
 247         vq_update_avail_ring(txvq, head_idx);
 248
 249         return 0;
 250 }
 251
 252 static inline struct rte_mbuf *
 253 rte_rxmbuf_alloc(struct rte_mempool *mp)
 254 {
 255         struct rte_mbuf *m;
 256
 257         m = __rte_mbuf_raw_alloc(mp);
 258         __rte_mbuf_sanity_check_raw(m, 0);
 259
 260         return m;
 261 }
 262
 263 static void
 264 virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
 265 {
 266         struct rte_mbuf *m;
 267         int i, nbufs, error, size = vq->vq_nentries;
 268         struct vring *vr = &vq->vq_ring;
 269         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 270
 271         PMD_INIT_FUNC_TRACE();
 272
 273         /*
 274          * Reinitialise since virtio port might have been stopped and restarted
 275          */
 276         memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size);
 277         vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
 278         vq->vq_used_cons_idx = 0;
 279         vq->vq_desc_head_idx = 0;
 280         vq->vq_avail_idx = 0;
 281         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 282         vq->vq_free_cnt = vq->vq_nentries;
 283         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 284
 285         /* Chain all the descriptors in the ring with an END */
 286         for (i = 0; i < size - 1; i++)
 287                 vr->desc[i].next = (uint16_t)(i + 1);
 288         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 289
 290         /*
 291          * Disable device(host) interrupting guest
 292          */
 293         virtqueue_disable_intr(vq);
 294
 295         /* Only rx virtqueue needs mbufs to be allocated at initialization */
 296         if (queue_type == VTNET_RQ) {
 297                 if (vq->mpool == NULL)
 298                         rte_exit(EXIT_FAILURE,
 299                         "Cannot allocate initial mbufs for rx virtqueue");
 300
 301                 /* Allocate blank mbufs for the each rx descriptor */
 302                 nbufs = 0;
 303                 error = ENOSPC;
 304
 305                 if (use_simple_rxtx)
 306                         for (i = 0; i < vq->vq_nentries; i++) {
 307                                 vq->vq_ring.avail->ring[i] = i;
 308                                 vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
 309                         }
 310
 311                 memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf));
 312                 for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++)
 313                         vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf;
 314
 315                 while (!virtqueue_full(vq)) {
 316                         m = rte_rxmbuf_alloc(vq->mpool);
 317                         if (m == NULL)
 318                                 break;
 319
 320                         /******************************************
 321                         *         Enqueue allocated buffers        *
 322                         *******************************************/
 323                         if (use_simple_rxtx)
 324                                 error = virtqueue_enqueue_recv_refill_simple(vq, m);
 325                         else
 326                                 error = virtqueue_enqueue_recv_refill(vq, m);
 327                         if (error) {
 328                                 rte_pktmbuf_free(m);
 329                                 break;
 330                         }
 331                         nbufs++;
 332                 }
 333
 334                 vq_update_avail_idx(vq);
 335
 336                 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
 337
 338                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 339                         vq->vq_queue_index);
 340                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 341                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 342         } else if (queue_type == VTNET_TQ) {
 343                 if (use_simple_rxtx) {
 344                         int mid_idx  = vq->vq_nentries >> 1;
 345                         for (i = 0; i < mid_idx; i++) {
 346                                 vq->vq_ring.avail->ring[i] = i + mid_idx;
 347                                 vq->vq_ring.desc[i + mid_idx].next = i;
 348                                 vq->vq_ring.desc[i + mid_idx].addr =
 349                                         vq->virtio_net_hdr_mem +
 350                                                 mid_idx * vq->hw->vtnet_hdr_size;
 351                                 vq->vq_ring.desc[i + mid_idx].len =
 352                                         vq->hw->vtnet_hdr_size;
 353                                 vq->vq_ring.desc[i + mid_idx].flags =
 354                                         VRING_DESC_F_NEXT;
 355                                 vq->vq_ring.desc[i].flags = 0;
 356                         }
 357                         for (i = mid_idx; i < vq->vq_nentries; i++)
 358                                 vq->vq_ring.avail->ring[i] = i;
 359                 }
 360
 361                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 362                         vq->vq_queue_index);
 363                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 364                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 365         } else {
 366                 VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL,
 367                         vq->vq_queue_index);
 368                 VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN,
 369                         vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 370         }
 371 }
 372
 373 void
 374 virtio_dev_cq_start(struct rte_eth_dev *dev)
 375 {
 376         struct virtio_hw *hw = dev->data->dev_private;
 377
 378         if (hw->cvq) {
 379                 virtio_dev_vring_start(hw->cvq, VTNET_CQ);
 380                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
 381         }
 382 }
 383
 384 void
 385 virtio_dev_rxtx_start(struct rte_eth_dev *dev)
 386 {
 387         /*
 388          * Start receive and transmit vrings
 389          * -    Setup vring structure for all queues
 390          * -    Initialize descriptor for the rx vring
 391          * -    Allocate blank mbufs for the each rx descriptor
 392          *
 393          */
 394         int i;
 395
 396         PMD_INIT_FUNC_TRACE();
 397
 398         /* Start rx vring. */
 399         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 400                 virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ);
 401                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 402         }
 403
 404         /* Start tx vring. */
 405         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 406                 virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ);
 407                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 408         }
 409 }
 410
 411 int
 412 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 413                         uint16_t queue_idx,
 414                         uint16_t nb_desc,
 415                         unsigned int socket_id,
 416                         __rte_unused const struct rte_eth_rxconf *rx_conf,
 417                         struct rte_mempool *mp)
 418 {
 419         uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 420         struct virtqueue *vq;
 421         int ret;
 422
 423         PMD_INIT_FUNC_TRACE();
 424         ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
 425                         nb_desc, socket_id, &vq);
 426         if (ret < 0) {
 427                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 428                 return ret;
 429         }
 430
 431         /* Create mempool for rx mbuf allocation */
 432         vq->mpool = mp;
 433
 434         dev->data->rx_queues[queue_idx] = vq;
 435
 436         virtio_rxq_vec_setup(vq);
 437
 438         return 0;
 439 }
 440
 441 void
 442 virtio_dev_rx_queue_release(void *rxq)
 443 {
 444         virtio_dev_queue_release(rxq);
 445 }
 446
 447 /*
 448  * struct rte_eth_dev *dev: Used to update dev
 449  * uint16_t nb_desc: Defaults to values read from config space
 450  * unsigned int socket_id: Used to allocate memzone
 451  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
 452  * uint16_t queue_idx: Just used as an index in dev txq list
 453  */
 454 int
 455 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 456                         uint16_t queue_idx,
 457                         uint16_t nb_desc,
 458                         unsigned int socket_id,
 459                         const struct rte_eth_txconf *tx_conf)
 460 {
 461         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 462         struct virtqueue *vq;
 463         uint16_t tx_free_thresh;
 464         int ret;
 465
 466         PMD_INIT_FUNC_TRACE();
 467
 468         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS)
 469             != ETH_TXQ_FLAGS_NOXSUMS) {
 470                 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n");
 471                 return -EINVAL;
 472         }
 473
 474         ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
 475                         nb_desc, socket_id, &vq);
 476         if (ret < 0) {
 477                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 478                 return ret;
 479         }
 480
 481         tx_free_thresh = tx_conf->tx_free_thresh;
 482         if (tx_free_thresh == 0)
 483                 tx_free_thresh =
 484                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
 485
 486         if (tx_free_thresh >= (vq->vq_nentries - 3)) {
 487                 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the "
 488                         "number of TX entries minus 3 (%u)."
 489                         " (tx_free_thresh=%u port=%u queue=%u)\n",
 490                         vq->vq_nentries - 3,
 491                         tx_free_thresh, dev->data->port_id, queue_idx);
 492                 return -EINVAL;
 493         }
 494
 495         vq->vq_free_thresh = tx_free_thresh;
 496
 497         dev->data->tx_queues[queue_idx] = vq;
 498         return 0;
 499 }
 500
 501 void
 502 virtio_dev_tx_queue_release(void *txq)
 503 {
 504         virtio_dev_queue_release(txq);
 505 }
 506
 507 static void
 508 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 509 {
 510         int error;
 511         /*
 512          * Requeue the discarded mbuf. This should always be
 513          * successful since it was just dequeued.
 514          */
 515         error = virtqueue_enqueue_recv_refill(vq, m);
 516         if (unlikely(error)) {
 517                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
 518                 rte_pktmbuf_free(m);
 519         }
 520 }
 521
 522 #define VIRTIO_MBUF_BURST_SZ 64
 523 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 524 uint16_t
 525 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 526 {
 527         struct virtqueue *rxvq = rx_queue;
 528         struct virtio_hw *hw;
 529         struct rte_mbuf *rxm, *new_mbuf;
 530         uint16_t nb_used, num, nb_rx;
 531         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 532         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 533         int error;
 534         uint32_t i, nb_enqueued;
 535         const uint32_t hdr_size = sizeof(struct virtio_net_hdr);
 536
 537         nb_used = VIRTQUEUE_NUSED(rxvq);
 538
 539         virtio_rmb();
 540
 541         num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
 542         num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
 543         if (likely(num > DESC_PER_CACHELINE))
 544                 num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
 545
 546         if (num == 0)
 547                 return 0;
 548
 549         num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num);
 550         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
 551
 552         hw = rxvq->hw;
 553         nb_rx = 0;
 554         nb_enqueued = 0;
 555
 556         for (i = 0; i < num ; i++) {
 557                 rxm = rcv_pkts[i];
 558
 559                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
 560
 561                 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
 562                         PMD_RX_LOG(ERR, "Packet drop");
 563                         nb_enqueued++;
 564                         virtio_discard_rxbuf(rxvq, rxm);
 565                         rxvq->errors++;
 566                         continue;
 567                 }
 568
 569                 rxm->port = rxvq->port_id;
 570                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 571
 572                 rxm->nb_segs = 1;
 573                 rxm->next = NULL;
 574                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
 575                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
 576
 577                 if (hw->vlan_strip)
 578                         rte_vlan_strip(rxm);
 579
 580                 VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
 581
 582                 rx_pkts[nb_rx++] = rxm;
 583                 rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
 584         }
 585
 586         rxvq->packets += nb_rx;
 587
 588         /* Allocate new mbuf for the used descriptor */
 589         error = ENOSPC;
 590         while (likely(!virtqueue_full(rxvq))) {
 591                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 592                 if (unlikely(new_mbuf == NULL)) {
 593                         struct rte_eth_dev *dev
 594                                 = &rte_eth_devices[rxvq->port_id];
 595                         dev->data->rx_mbuf_alloc_failed++;
 596                         break;
 597                 }
 598                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 599                 if (unlikely(error)) {
 600                         rte_pktmbuf_free(new_mbuf);
 601                         break;
 602                 }
 603                 nb_enqueued++;
 604         }
 605
 606         if (likely(nb_enqueued)) {
 607                 vq_update_avail_idx(rxvq);
 608
 609                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 610                         virtqueue_notify(rxvq);
 611                         PMD_RX_LOG(DEBUG, "Notified\n");
 612                 }
 613         }
 614
 615         return nb_rx;
 616 }
 617
 618 uint16_t
 619 virtio_recv_mergeable_pkts(void *rx_queue,
 620                         struct rte_mbuf **rx_pkts,
 621                         uint16_t nb_pkts)
 622 {
 623         struct virtqueue *rxvq = rx_queue;
 624         struct virtio_hw *hw;
 625         struct rte_mbuf *rxm, *new_mbuf;
 626         uint16_t nb_used, num, nb_rx;
 627         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 628         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 629         struct rte_mbuf *prev;
 630         int error;
 631         uint32_t i, nb_enqueued;
 632         uint32_t seg_num;
 633         uint16_t extra_idx;
 634         uint32_t seg_res;
 635         const uint32_t hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 636
 637         nb_used = VIRTQUEUE_NUSED(rxvq);
 638
 639         virtio_rmb();
 640
 641         if (nb_used == 0)
 642                 return 0;
 643
 644         PMD_RX_LOG(DEBUG, "used:%d\n", nb_used);
 645
 646         hw = rxvq->hw;
 647         nb_rx = 0;
 648         i = 0;
 649         nb_enqueued = 0;
 650         seg_num = 0;
 651         extra_idx = 0;
 652         seg_res = 0;
 653
 654         while (i < nb_used) {
 655                 struct virtio_net_hdr_mrg_rxbuf *header;
 656
 657                 if (nb_rx == nb_pkts)
 658                         break;
 659
 660                 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1);
 661                 if (num != 1)
 662                         continue;
 663
 664                 i++;
 665
 666                 PMD_RX_LOG(DEBUG, "dequeue:%d\n", num);
 667                 PMD_RX_LOG(DEBUG, "packet len:%d\n", len[0]);
 668
 669                 rxm = rcv_pkts[0];
 670
 671                 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
 672                         PMD_RX_LOG(ERR, "Packet drop\n");
 673                         nb_enqueued++;
 674                         virtio_discard_rxbuf(rxvq, rxm);
 675                         rxvq->errors++;
 676                         continue;
 677                 }
 678
 679                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
 680                         RTE_PKTMBUF_HEADROOM - hdr_size);
 681                 seg_num = header->num_buffers;
 682
 683                 if (seg_num == 0)
 684                         seg_num = 1;
 685
 686                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 687                 rxm->nb_segs = seg_num;
 688                 rxm->next = NULL;
 689                 rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
 690                 rxm->data_len = (uint16_t)(len[0] - hdr_size);
 691
 692                 rxm->port = rxvq->port_id;
 693                 rx_pkts[nb_rx] = rxm;
 694                 prev = rxm;
 695
 696                 seg_res = seg_num - 1;
 697
 698                 while (seg_res != 0) {
 699                         /*
 700                          * Get extra segments for current uncompleted packet.
 701                          */
 702                         uint16_t  rcv_cnt =
 703                                 RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
 704                         if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) {
 705                                 uint32_t rx_num =
 706                                         virtqueue_dequeue_burst_rx(rxvq,
 707                                         rcv_pkts, len, rcv_cnt);
 708                                 i += rx_num;
 709                                 rcv_cnt = rx_num;
 710                         } else {
 711                                 PMD_RX_LOG(ERR,
 712                                         "No enough segments for packet.\n");
 713                                 nb_enqueued++;
 714                                 virtio_discard_rxbuf(rxvq, rxm);
 715                                 rxvq->errors++;
 716                                 break;
 717                         }
 718
 719                         extra_idx = 0;
 720
 721                         while (extra_idx < rcv_cnt) {
 722                                 rxm = rcv_pkts[extra_idx];
 723
 724                                 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
 725                                 rxm->next = NULL;
 726                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
 727                                 rxm->data_len = (uint16_t)(len[extra_idx]);
 728
 729                                 if (prev)
 730                                         prev->next = rxm;
 731
 732                                 prev = rxm;
 733                                 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len;
 734                                 extra_idx++;
 735                         };
 736                         seg_res -= rcv_cnt;
 737                 }
 738
 739                 if (hw->vlan_strip)
 740                         rte_vlan_strip(rx_pkts[nb_rx]);
 741
 742                 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
 743                         rx_pkts[nb_rx]->data_len);
 744
 745                 rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
 746                 nb_rx++;
 747         }
 748
 749         rxvq->packets += nb_rx;
 750
 751         /* Allocate new mbuf for the used descriptor */
 752         error = ENOSPC;
 753         while (likely(!virtqueue_full(rxvq))) {
 754                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 755                 if (unlikely(new_mbuf == NULL)) {
 756                         struct rte_eth_dev *dev
 757                                 = &rte_eth_devices[rxvq->port_id];
 758                         dev->data->rx_mbuf_alloc_failed++;
 759                         break;
 760                 }
 761                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 762                 if (unlikely(error)) {
 763                         rte_pktmbuf_free(new_mbuf);
 764                         break;
 765                 }
 766                 nb_enqueued++;
 767         }
 768
 769         if (likely(nb_enqueued)) {
 770                 vq_update_avail_idx(rxvq);
 771
 772                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 773                         virtqueue_notify(rxvq);
 774                         PMD_RX_LOG(DEBUG, "Notified");
 775                 }
 776         }
 777
 778         return nb_rx;
 779 }
 780
 781 uint16_t
 782 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 783 {
 784         struct virtqueue *txvq = tx_queue;
 785         struct rte_mbuf *txm;
 786         uint16_t nb_used, nb_tx;
 787         int error;
 788
 789         if (unlikely(nb_pkts < 1))
 790                 return nb_pkts;
 791
 792         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 793         nb_used = VIRTQUEUE_NUSED(txvq);
 794
 795         virtio_rmb();
 796         if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh))
 797                 virtio_xmit_cleanup(txvq, nb_used);
 798
 799         nb_tx = 0;
 800
 801         while (nb_tx < nb_pkts) {
 802                 /* Need one more descriptor for virtio header. */
 803                 int need = tx_pkts[nb_tx]->nb_segs - txvq->vq_free_cnt + 1;
 804
 805                 /*Positive value indicates it need free vring descriptors */
 806                 if (unlikely(need > 0)) {
 807                         nb_used = VIRTQUEUE_NUSED(txvq);
 808                         virtio_rmb();
 809                         need = RTE_MIN(need, (int)nb_used);
 810
 811                         virtio_xmit_cleanup(txvq, need);
 812                         need = (int)tx_pkts[nb_tx]->nb_segs -
 813                                 txvq->vq_free_cnt + 1;
 814                 }
 815
 816                 /*
 817                  * Zero or negative value indicates it has enough free
 818                  * descriptors to use for transmitting.
 819                  */
 820                 if (likely(need <= 0)) {
 821                         txm = tx_pkts[nb_tx];
 822
 823                         /* Do VLAN tag insertion */
 824                         if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
 825                                 error = rte_vlan_insert(&txm);
 826                                 if (unlikely(error)) {
 827                                         rte_pktmbuf_free(txm);
 828                                         ++nb_tx;
 829                                         continue;
 830                                 }
 831                         }
 832
 833                         /* Enqueue Packet buffers */
 834                         error = virtqueue_enqueue_xmit(txvq, txm);
 835                         if (unlikely(error)) {
 836                                 if (error == ENOSPC)
 837                                         PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0");
 838                                 else if (error == EMSGSIZE)
 839                                         PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1");
 840                                 else
 841                                         PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error);
 842                                 break;
 843                         }
 844                         nb_tx++;
 845                         txvq->bytes += txm->pkt_len;
 846                 } else {
 847                         PMD_TX_LOG(ERR, "No free tx descriptors to transmit");
 848                         break;
 849                 }
 850         }
 851
 852         txvq->packets += nb_tx;
 853
 854         if (likely(nb_tx)) {
 855                 vq_update_avail_idx(txvq);
 856
 857                 if (unlikely(virtqueue_kick_prepare(txvq))) {
 858                         virtqueue_notify(txvq);
 859                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
 860                 }
 861         }
 862
 863         return nb_tx;
 864 }