drivers/net/virtio/virtio_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 #include <rte_cycles.h>
  41 #include <rte_memory.h>
  42 #include <rte_memzone.h>
  43 #include <rte_branch_prediction.h>
  44 #include <rte_mempool.h>
  45 #include <rte_malloc.h>
  46 #include <rte_mbuf.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_prefetch.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_errno.h>
  52 #include <rte_byteorder.h>
  53
  54 #include "virtio_logs.h"
  55 #include "virtio_ethdev.h"
  56 #include "virtio_pci.h"
  57 #include "virtqueue.h"
  58 #include "virtio_rxtx.h"
  59
  60 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
  61 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
  62 #else
  63 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
  64 #endif
  65
  66
  67 #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
  68         ETH_TXQ_FLAGS_NOOFFLOADS)
  69
  70 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
  71 static int use_simple_rxtx;
  72 #endif
  73
  74 static void
  75 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  76 {
  77         struct vring_desc *dp, *dp_tail;
  78         struct vq_desc_extra *dxp;
  79         uint16_t desc_idx_last = desc_idx;
  80
  81         dp  = &vq->vq_ring.desc[desc_idx];
  82         dxp = &vq->vq_descx[desc_idx];
  83         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
  84         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
  85                 while (dp->flags & VRING_DESC_F_NEXT) {
  86                         desc_idx_last = dp->next;
  87                         dp = &vq->vq_ring.desc[dp->next];
  88                 }
  89         }
  90         dxp->ndescs = 0;
  91
  92         /*
  93          * We must append the existing free chain, if any, to the end of
  94          * newly freed chain. If the virtqueue was completely used, then
  95          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  96          */
  97         if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
  98                 vq->vq_desc_head_idx = desc_idx;
  99         } else {
 100                 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
 101                 dp_tail->next = desc_idx;
 102         }
 103
 104         vq->vq_desc_tail_idx = desc_idx_last;
 105         dp->next = VQ_RING_DESC_CHAIN_END;
 106 }
 107
 108 static uint16_t
 109 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 110                            uint32_t *len, uint16_t num)
 111 {
 112         struct vring_used_elem *uep;
 113         struct rte_mbuf *cookie;
 114         uint16_t used_idx, desc_idx;
 115         uint16_t i;
 116
 117         /*  Caller does the check */
 118         for (i = 0; i < num ; i++) {
 119                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 120                 uep = &vq->vq_ring.used->ring[used_idx];
 121                 desc_idx = (uint16_t) uep->id;
 122                 len[i] = uep->len;
 123                 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
 124
 125                 if (unlikely(cookie == NULL)) {
 126                         PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
 127                                 vq->vq_used_cons_idx);
 128                         break;
 129                 }
 130
 131                 rte_prefetch0(cookie);
 132                 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
 133                 rx_pkts[i]  = cookie;
 134                 vq->vq_used_cons_idx++;
 135                 vq_ring_free_chain(vq, desc_idx);
 136                 vq->vq_descx[desc_idx].cookie = NULL;
 137         }
 138
 139         return i;
 140 }
 141
 142 #ifndef DEFAULT_TX_FREE_THRESH
 143 #define DEFAULT_TX_FREE_THRESH 32
 144 #endif
 145
 146 /* Cleanup from completed transmits. */
 147 static void
 148 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
 149 {
 150         uint16_t i, used_idx, desc_idx;
 151         for (i = 0; i < num; i++) {
 152                 struct vring_used_elem *uep;
 153                 struct vq_desc_extra *dxp;
 154
 155                 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
 156                 uep = &vq->vq_ring.used->ring[used_idx];
 157
 158                 desc_idx = (uint16_t) uep->id;
 159                 dxp = &vq->vq_descx[desc_idx];
 160                 vq->vq_used_cons_idx++;
 161                 vq_ring_free_chain(vq, desc_idx);
 162
 163                 if (dxp->cookie != NULL) {
 164                         rte_pktmbuf_free(dxp->cookie);
 165                         dxp->cookie = NULL;
 166                 }
 167         }
 168 }
 169
 170
 171 static inline int
 172 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
 173 {
 174         struct vq_desc_extra *dxp;
 175         struct virtio_hw *hw = vq->hw;
 176         struct vring_desc *start_dp;
 177         uint16_t needed = 1;
 178         uint16_t head_idx, idx;
 179
 180         if (unlikely(vq->vq_free_cnt == 0))
 181                 return -ENOSPC;
 182         if (unlikely(vq->vq_free_cnt < needed))
 183                 return -EMSGSIZE;
 184
 185         head_idx = vq->vq_desc_head_idx;
 186         if (unlikely(head_idx >= vq->vq_nentries))
 187                 return -EFAULT;
 188
 189         idx = head_idx;
 190         dxp = &vq->vq_descx[idx];
 191         dxp->cookie = (void *)cookie;
 192         dxp->ndescs = needed;
 193
 194         start_dp = vq->vq_ring.desc;
 195         start_dp[idx].addr =
 196                 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM
 197                 - hw->vtnet_hdr_size);
 198         start_dp[idx].len =
 199                 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
 200         start_dp[idx].flags =  VRING_DESC_F_WRITE;
 201         idx = start_dp[idx].next;
 202         vq->vq_desc_head_idx = idx;
 203         if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 204                 vq->vq_desc_tail_idx = idx;
 205         vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
 206         vq_update_avail_ring(vq, head_idx);
 207
 208         return 0;
 209 }
 210
 211 static int
 212 virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
 213 {
 214         struct vq_desc_extra *dxp;
 215         struct vring_desc *start_dp;
 216         uint16_t seg_num = cookie->nb_segs;
 217         uint16_t needed = 1 + seg_num;
 218         uint16_t head_idx, idx;
 219         size_t head_size = txvq->hw->vtnet_hdr_size;
 220
 221         if (unlikely(txvq->vq_free_cnt == 0))
 222                 return -ENOSPC;
 223         if (unlikely(txvq->vq_free_cnt < needed))
 224                 return -EMSGSIZE;
 225         head_idx = txvq->vq_desc_head_idx;
 226         if (unlikely(head_idx >= txvq->vq_nentries))
 227                 return -EFAULT;
 228
 229         idx = head_idx;
 230         dxp = &txvq->vq_descx[idx];
 231         dxp->cookie = (void *)cookie;
 232         dxp->ndescs = needed;
 233
 234         start_dp = txvq->vq_ring.desc;
 235         start_dp[idx].addr =
 236                 txvq->virtio_net_hdr_mem + idx * head_size;
 237         start_dp[idx].len = head_size;
 238         start_dp[idx].flags = VRING_DESC_F_NEXT;
 239
 240         for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) {
 241                 idx = start_dp[idx].next;
 242                 start_dp[idx].addr  = rte_mbuf_data_dma_addr(cookie);
 243                 start_dp[idx].len   = cookie->data_len;
 244                 start_dp[idx].flags = VRING_DESC_F_NEXT;
 245                 cookie = cookie->next;
 246         }
 247
 248         start_dp[idx].flags &= ~VRING_DESC_F_NEXT;
 249         idx = start_dp[idx].next;
 250         txvq->vq_desc_head_idx = idx;
 251         if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 252                 txvq->vq_desc_tail_idx = idx;
 253         txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
 254         vq_update_avail_ring(txvq, head_idx);
 255
 256         return 0;
 257 }
 258
 259 static inline struct rte_mbuf *
 260 rte_rxmbuf_alloc(struct rte_mempool *mp)
 261 {
 262         struct rte_mbuf *m;
 263
 264         m = __rte_mbuf_raw_alloc(mp);
 265         __rte_mbuf_sanity_check_raw(m, 0);
 266
 267         return m;
 268 }
 269
 270 static void
 271 virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
 272 {
 273         struct rte_mbuf *m;
 274         int i, nbufs, error, size = vq->vq_nentries;
 275         struct vring *vr = &vq->vq_ring;
 276         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 277
 278         PMD_INIT_FUNC_TRACE();
 279
 280         /*
 281          * Reinitialise since virtio port might have been stopped and restarted
 282          */
 283         memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size);
 284         vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
 285         vq->vq_used_cons_idx = 0;
 286         vq->vq_desc_head_idx = 0;
 287         vq->vq_avail_idx = 0;
 288         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 289         vq->vq_free_cnt = vq->vq_nentries;
 290         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 291
 292         /* Chain all the descriptors in the ring with an END */
 293         for (i = 0; i < size - 1; i++)
 294                 vr->desc[i].next = (uint16_t)(i + 1);
 295         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 296
 297         /*
 298          * Disable device(host) interrupting guest
 299          */
 300         virtqueue_disable_intr(vq);
 301
 302         /* Only rx virtqueue needs mbufs to be allocated at initialization */
 303         if (queue_type == VTNET_RQ) {
 304                 if (vq->mpool == NULL)
 305                         rte_exit(EXIT_FAILURE,
 306                         "Cannot allocate initial mbufs for rx virtqueue");
 307
 308                 /* Allocate blank mbufs for the each rx descriptor */
 309                 nbufs = 0;
 310                 error = ENOSPC;
 311
 312 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
 313                 if (use_simple_rxtx)
 314                         for (i = 0; i < vq->vq_nentries; i++) {
 315                                 vq->vq_ring.avail->ring[i] = i;
 316                                 vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
 317                         }
 318 #endif
 319                 memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf));
 320                 for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++)
 321                         vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf;
 322
 323                 while (!virtqueue_full(vq)) {
 324                         m = rte_rxmbuf_alloc(vq->mpool);
 325                         if (m == NULL)
 326                                 break;
 327
 328                         /******************************************
 329                         *         Enqueue allocated buffers        *
 330                         *******************************************/
 331 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
 332                         if (use_simple_rxtx)
 333                                 error = virtqueue_enqueue_recv_refill_simple(vq, m);
 334                         else
 335 #endif
 336                                 error = virtqueue_enqueue_recv_refill(vq, m);
 337                         if (error) {
 338                                 rte_pktmbuf_free(m);
 339                                 break;
 340                         }
 341                         nbufs++;
 342                 }
 343
 344                 vq_update_avail_idx(vq);
 345
 346                 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
 347         } else if (queue_type == VTNET_TQ) {
 348 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
 349                 if (use_simple_rxtx) {
 350                         int mid_idx  = vq->vq_nentries >> 1;
 351                         for (i = 0; i < mid_idx; i++) {
 352                                 vq->vq_ring.avail->ring[i] = i + mid_idx;
 353                                 vq->vq_ring.desc[i + mid_idx].next = i;
 354                                 vq->vq_ring.desc[i + mid_idx].addr =
 355                                         vq->virtio_net_hdr_mem +
 356                                                 i * vq->hw->vtnet_hdr_size;
 357                                 vq->vq_ring.desc[i + mid_idx].len =
 358                                         vq->hw->vtnet_hdr_size;
 359                                 vq->vq_ring.desc[i + mid_idx].flags =
 360                                         VRING_DESC_F_NEXT;
 361                                 vq->vq_ring.desc[i].flags = 0;
 362                         }
 363                         for (i = mid_idx; i < vq->vq_nentries; i++)
 364                                 vq->vq_ring.avail->ring[i] = i;
 365                 }
 366 #endif
 367         }
 368 }
 369
 370 void
 371 virtio_dev_cq_start(struct rte_eth_dev *dev)
 372 {
 373         struct virtio_hw *hw = dev->data->dev_private;
 374
 375         if (hw->cvq) {
 376                 virtio_dev_vring_start(hw->cvq, VTNET_CQ);
 377                 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
 378         }
 379 }
 380
 381 void
 382 virtio_dev_rxtx_start(struct rte_eth_dev *dev)
 383 {
 384         /*
 385          * Start receive and transmit vrings
 386          * -    Setup vring structure for all queues
 387          * -    Initialize descriptor for the rx vring
 388          * -    Allocate blank mbufs for the each rx descriptor
 389          *
 390          */
 391         int i;
 392
 393         PMD_INIT_FUNC_TRACE();
 394
 395         /* Start rx vring. */
 396         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 397                 virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ);
 398                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 399         }
 400
 401         /* Start tx vring. */
 402         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 403                 virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ);
 404                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 405         }
 406 }
 407
 408 int
 409 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 410                         uint16_t queue_idx,
 411                         uint16_t nb_desc,
 412                         unsigned int socket_id,
 413                         __rte_unused const struct rte_eth_rxconf *rx_conf,
 414                         struct rte_mempool *mp)
 415 {
 416         uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
 417         struct virtqueue *vq;
 418         int ret;
 419
 420         PMD_INIT_FUNC_TRACE();
 421         ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
 422                         nb_desc, socket_id, &vq);
 423         if (ret < 0) {
 424                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 425                 return ret;
 426         }
 427
 428         /* Create mempool for rx mbuf allocation */
 429         vq->mpool = mp;
 430
 431         dev->data->rx_queues[queue_idx] = vq;
 432
 433 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
 434         virtio_rxq_vec_setup(vq);
 435 #endif
 436
 437         return 0;
 438 }
 439
 440 void
 441 virtio_dev_rx_queue_release(void *rxq)
 442 {
 443         virtio_dev_queue_release(rxq);
 444 }
 445
 446 /*
 447  * struct rte_eth_dev *dev: Used to update dev
 448  * uint16_t nb_desc: Defaults to values read from config space
 449  * unsigned int socket_id: Used to allocate memzone
 450  * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
 451  * uint16_t queue_idx: Just used as an index in dev txq list
 452  */
 453 int
 454 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 455                         uint16_t queue_idx,
 456                         uint16_t nb_desc,
 457                         unsigned int socket_id,
 458                         const struct rte_eth_txconf *tx_conf)
 459 {
 460         uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 461
 462 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
 463         struct virtio_hw *hw = dev->data->dev_private;
 464 #endif
 465         struct virtqueue *vq;
 466         uint16_t tx_free_thresh;
 467         int ret;
 468
 469         PMD_INIT_FUNC_TRACE();
 470
 471         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS)
 472             != ETH_TXQ_FLAGS_NOXSUMS) {
 473                 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n");
 474                 return -EINVAL;
 475         }
 476
 477 #ifdef RTE_MACHINE_CPUFLAG_SSSE3
 478         /* Use simple rx/tx func if single segment and no offloads */
 479         if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS &&
 480              !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
 481                 PMD_INIT_LOG(INFO, "Using simple rx/tx path");
 482                 dev->tx_pkt_burst = virtio_xmit_pkts_simple;
 483                 dev->rx_pkt_burst = virtio_recv_pkts_vec;
 484                 use_simple_rxtx = 1;
 485         }
 486 #endif
 487
 488         ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
 489                         nb_desc, socket_id, &vq);
 490         if (ret < 0) {
 491                 PMD_INIT_LOG(ERR, "rvq initialization failed");
 492                 return ret;
 493         }
 494
 495         tx_free_thresh = tx_conf->tx_free_thresh;
 496         if (tx_free_thresh == 0)
 497                 tx_free_thresh =
 498                         RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
 499
 500         if (tx_free_thresh >= (vq->vq_nentries - 3)) {
 501                 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the "
 502                         "number of TX entries minus 3 (%u)."
 503                         " (tx_free_thresh=%u port=%u queue=%u)\n",
 504                         vq->vq_nentries - 3,
 505                         tx_free_thresh, dev->data->port_id, queue_idx);
 506                 return -EINVAL;
 507         }
 508
 509         vq->vq_free_thresh = tx_free_thresh;
 510
 511         dev->data->tx_queues[queue_idx] = vq;
 512         return 0;
 513 }
 514
 515 void
 516 virtio_dev_tx_queue_release(void *txq)
 517 {
 518         virtio_dev_queue_release(txq);
 519 }
 520
 521 static void
 522 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 523 {
 524         int error;
 525         /*
 526          * Requeue the discarded mbuf. This should always be
 527          * successful since it was just dequeued.
 528          */
 529         error = virtqueue_enqueue_recv_refill(vq, m);
 530         if (unlikely(error)) {
 531                 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
 532                 rte_pktmbuf_free(m);
 533         }
 534 }
 535
 536 static void
 537 virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf)
 538 {
 539         uint32_t s = mbuf->pkt_len;
 540         struct ether_addr *ea;
 541
 542         if (s == 64) {
 543                 vq->size_bins[1]++;
 544         } else if (s > 64 && s < 1024) {
 545                 uint32_t bin;
 546
 547                 /* count zeros, and offset into correct bin */
 548                 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
 549                 vq->size_bins[bin]++;
 550         } else {
 551                 if (s < 64)
 552                         vq->size_bins[0]++;
 553                 else if (s < 1519)
 554                         vq->size_bins[6]++;
 555                 else if (s >= 1519)
 556                         vq->size_bins[7]++;
 557         }
 558
 559         ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
 560         if (is_multicast_ether_addr(ea)) {
 561                 if (is_broadcast_ether_addr(ea))
 562                         vq->broadcast++;
 563                 else
 564                         vq->multicast++;
 565         }
 566 }
 567
 568 #define VIRTIO_MBUF_BURST_SZ 64
 569 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 570 uint16_t
 571 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 572 {
 573         struct virtqueue *rxvq = rx_queue;
 574         struct virtio_hw *hw;
 575         struct rte_mbuf *rxm, *new_mbuf;
 576         uint16_t nb_used, num, nb_rx;
 577         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 578         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 579         int error;
 580         uint32_t i, nb_enqueued;
 581         uint32_t hdr_size;
 582
 583         nb_used = VIRTQUEUE_NUSED(rxvq);
 584
 585         virtio_rmb();
 586
 587         num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
 588         num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
 589         if (likely(num > DESC_PER_CACHELINE))
 590                 num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
 591
 592         if (num == 0)
 593                 return 0;
 594
 595         num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num);
 596         PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
 597
 598         hw = rxvq->hw;
 599         nb_rx = 0;
 600         nb_enqueued = 0;
 601         hdr_size = hw->vtnet_hdr_size;
 602
 603         for (i = 0; i < num ; i++) {
 604                 rxm = rcv_pkts[i];
 605
 606                 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
 607
 608                 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
 609                         PMD_RX_LOG(ERR, "Packet drop");
 610                         nb_enqueued++;
 611                         virtio_discard_rxbuf(rxvq, rxm);
 612                         rxvq->errors++;
 613                         continue;
 614                 }
 615
 616                 rxm->port = rxvq->port_id;
 617                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 618                 rxm->ol_flags = 0;
 619                 rxm->vlan_tci = 0;
 620
 621                 rxm->nb_segs = 1;
 622                 rxm->next = NULL;
 623                 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
 624                 rxm->data_len = (uint16_t)(len[i] - hdr_size);
 625
 626                 if (hw->vlan_strip)
 627                         rte_vlan_strip(rxm);
 628
 629                 VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
 630
 631                 rx_pkts[nb_rx++] = rxm;
 632
 633                 rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
 634                 virtio_update_packet_stats(rxvq, rxm);
 635         }
 636
 637         rxvq->packets += nb_rx;
 638
 639         /* Allocate new mbuf for the used descriptor */
 640         error = ENOSPC;
 641         while (likely(!virtqueue_full(rxvq))) {
 642                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 643                 if (unlikely(new_mbuf == NULL)) {
 644                         struct rte_eth_dev *dev
 645                                 = &rte_eth_devices[rxvq->port_id];
 646                         dev->data->rx_mbuf_alloc_failed++;
 647                         break;
 648                 }
 649                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 650                 if (unlikely(error)) {
 651                         rte_pktmbuf_free(new_mbuf);
 652                         break;
 653                 }
 654                 nb_enqueued++;
 655         }
 656
 657         if (likely(nb_enqueued)) {
 658                 vq_update_avail_idx(rxvq);
 659
 660                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 661                         virtqueue_notify(rxvq);
 662                         PMD_RX_LOG(DEBUG, "Notified\n");
 663                 }
 664         }
 665
 666         return nb_rx;
 667 }
 668
 669 uint16_t
 670 virtio_recv_mergeable_pkts(void *rx_queue,
 671                         struct rte_mbuf **rx_pkts,
 672                         uint16_t nb_pkts)
 673 {
 674         struct virtqueue *rxvq = rx_queue;
 675         struct virtio_hw *hw;
 676         struct rte_mbuf *rxm, *new_mbuf;
 677         uint16_t nb_used, num, nb_rx;
 678         uint32_t len[VIRTIO_MBUF_BURST_SZ];
 679         struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
 680         struct rte_mbuf *prev;
 681         int error;
 682         uint32_t i, nb_enqueued;
 683         uint32_t seg_num;
 684         uint16_t extra_idx;
 685         uint32_t seg_res;
 686         uint32_t hdr_size;
 687
 688         nb_used = VIRTQUEUE_NUSED(rxvq);
 689
 690         virtio_rmb();
 691
 692         if (nb_used == 0)
 693                 return 0;
 694
 695         PMD_RX_LOG(DEBUG, "used:%d\n", nb_used);
 696
 697         hw = rxvq->hw;
 698         nb_rx = 0;
 699         i = 0;
 700         nb_enqueued = 0;
 701         seg_num = 0;
 702         extra_idx = 0;
 703         seg_res = 0;
 704         hdr_size = hw->vtnet_hdr_size;
 705
 706         while (i < nb_used) {
 707                 struct virtio_net_hdr_mrg_rxbuf *header;
 708
 709                 if (nb_rx == nb_pkts)
 710                         break;
 711
 712                 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1);
 713                 if (num != 1)
 714                         continue;
 715
 716                 i++;
 717
 718                 PMD_RX_LOG(DEBUG, "dequeue:%d\n", num);
 719                 PMD_RX_LOG(DEBUG, "packet len:%d\n", len[0]);
 720
 721                 rxm = rcv_pkts[0];
 722
 723                 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
 724                         PMD_RX_LOG(ERR, "Packet drop\n");
 725                         nb_enqueued++;
 726                         virtio_discard_rxbuf(rxvq, rxm);
 727                         rxvq->errors++;
 728                         continue;
 729                 }
 730
 731                 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
 732                         RTE_PKTMBUF_HEADROOM - hdr_size);
 733                 seg_num = header->num_buffers;
 734
 735                 if (seg_num == 0)
 736                         seg_num = 1;
 737
 738                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 739                 rxm->nb_segs = seg_num;
 740                 rxm->next = NULL;
 741                 rxm->ol_flags = 0;
 742                 rxm->vlan_tci = 0;
 743                 rxm->pkt_len = (uint32_t)(len[0] - hdr_size);
 744                 rxm->data_len = (uint16_t)(len[0] - hdr_size);
 745
 746                 rxm->port = rxvq->port_id;
 747                 rx_pkts[nb_rx] = rxm;
 748                 prev = rxm;
 749
 750                 seg_res = seg_num - 1;
 751
 752                 while (seg_res != 0) {
 753                         /*
 754                          * Get extra segments for current uncompleted packet.
 755                          */
 756                         uint16_t  rcv_cnt =
 757                                 RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
 758                         if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) {
 759                                 uint32_t rx_num =
 760                                         virtqueue_dequeue_burst_rx(rxvq,
 761                                         rcv_pkts, len, rcv_cnt);
 762                                 i += rx_num;
 763                                 rcv_cnt = rx_num;
 764                         } else {
 765                                 PMD_RX_LOG(ERR,
 766                                         "No enough segments for packet.\n");
 767                                 nb_enqueued++;
 768                                 virtio_discard_rxbuf(rxvq, rxm);
 769                                 rxvq->errors++;
 770                                 break;
 771                         }
 772
 773                         extra_idx = 0;
 774
 775                         while (extra_idx < rcv_cnt) {
 776                                 rxm = rcv_pkts[extra_idx];
 777
 778                                 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
 779                                 rxm->next = NULL;
 780                                 rxm->pkt_len = (uint32_t)(len[extra_idx]);
 781                                 rxm->data_len = (uint16_t)(len[extra_idx]);
 782
 783                                 if (prev)
 784                                         prev->next = rxm;
 785
 786                                 prev = rxm;
 787                                 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len;
 788                                 extra_idx++;
 789                         };
 790                         seg_res -= rcv_cnt;
 791                 }
 792
 793                 if (hw->vlan_strip)
 794                         rte_vlan_strip(rx_pkts[nb_rx]);
 795
 796                 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
 797                         rx_pkts[nb_rx]->data_len);
 798
 799                 rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
 800                 virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]);
 801                 nb_rx++;
 802         }
 803
 804         rxvq->packets += nb_rx;
 805
 806         /* Allocate new mbuf for the used descriptor */
 807         error = ENOSPC;
 808         while (likely(!virtqueue_full(rxvq))) {
 809                 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
 810                 if (unlikely(new_mbuf == NULL)) {
 811                         struct rte_eth_dev *dev
 812                                 = &rte_eth_devices[rxvq->port_id];
 813                         dev->data->rx_mbuf_alloc_failed++;
 814                         break;
 815                 }
 816                 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
 817                 if (unlikely(error)) {
 818                         rte_pktmbuf_free(new_mbuf);
 819                         break;
 820                 }
 821                 nb_enqueued++;
 822         }
 823
 824         if (likely(nb_enqueued)) {
 825                 vq_update_avail_idx(rxvq);
 826
 827                 if (unlikely(virtqueue_kick_prepare(rxvq))) {
 828                         virtqueue_notify(rxvq);
 829                         PMD_RX_LOG(DEBUG, "Notified");
 830                 }
 831         }
 832
 833         return nb_rx;
 834 }
 835
 836 uint16_t
 837 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 838 {
 839         struct virtqueue *txvq = tx_queue;
 840         uint16_t nb_used, nb_tx;
 841         int error;
 842
 843         if (unlikely(nb_pkts < 1))
 844                 return nb_pkts;
 845
 846         PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 847         nb_used = VIRTQUEUE_NUSED(txvq);
 848
 849         virtio_rmb();
 850         if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh))
 851                 virtio_xmit_cleanup(txvq, nb_used);
 852
 853         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
 854                 struct rte_mbuf *txm = tx_pkts[nb_tx];
 855                 /* Need one more descriptor for virtio header. */
 856                 int need = txm->nb_segs - txvq->vq_free_cnt + 1;
 857
 858                 /* Positive value indicates it need free vring descriptors */
 859                 if (unlikely(need > 0)) {
 860                         nb_used = VIRTQUEUE_NUSED(txvq);
 861                         virtio_rmb();
 862                         need = RTE_MIN(need, (int)nb_used);
 863
 864                         virtio_xmit_cleanup(txvq, need);
 865                         need = txm->nb_segs - txvq->vq_free_cnt + 1;
 866                         if (unlikely(need > 0)) {
 867                                 PMD_TX_LOG(ERR,
 868                                            "No free tx descriptors to transmit");
 869                                 break;
 870                         }
 871                 }
 872
 873                 /* Do VLAN tag insertion */
 874                 if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
 875                         error = rte_vlan_insert(&txm);
 876                         if (unlikely(error)) {
 877                                 rte_pktmbuf_free(txm);
 878                                 continue;
 879                         }
 880                 }
 881
 882                 /* Enqueue Packet buffers */
 883                 error = virtqueue_enqueue_xmit(txvq, txm);
 884                 if (unlikely(error)) {
 885                         if (error == ENOSPC)
 886                                 PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0");
 887                         else if (error == EMSGSIZE)
 888                                 PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1");
 889                         else
 890                                 PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error);
 891                         break;
 892                 }
 893
 894                 txvq->bytes += txm->pkt_len;
 895                 virtio_update_packet_stats(txvq, txm);
 896         }
 897
 898         txvq->packets += nb_tx;
 899
 900         if (likely(nb_tx)) {
 901                 vq_update_avail_idx(txvq);
 902
 903                 if (unlikely(virtqueue_kick_prepare(txvq))) {
 904                         virtqueue_notify(txvq);
 905                         PMD_TX_LOG(DEBUG, "Notified backend after xmit");
 906                 }
 907         }
 908
 909         return nb_tx;
 910 }