lib/librte_vhost/vhost_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <linux/virtio_net.h>
  36
  37 #include <rte_mbuf.h>
  38 #include <rte_memcpy.h>
  39 #include <rte_virtio_net.h>
  40
  41 #include "vhost-net.h"
  42
  43 #define MAX_PKT_BURST 32
  44
  45 /**
  46  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
  47  * be received from the physical port or from another virtio device. A packet
  48  * count is returned to indicate the number of packets that are succesfully
  49  * added to the RX queue. This function works when the mbuf is scattered, but
  50  * it doesn't support the mergeable feature.
  51  */
  52 static inline uint32_t __attribute__((always_inline))
  53 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
  54         struct rte_mbuf **pkts, uint32_t count)
  55 {
  56         struct vhost_virtqueue *vq;
  57         struct vring_desc *desc;
  58         struct rte_mbuf *buff;
  59         /* The virtio_hdr is initialised to 0. */
  60         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0};
  61         uint64_t buff_addr = 0;
  62         uint64_t buff_hdr_addr = 0;
  63         uint32_t head[MAX_PKT_BURST];
  64         uint32_t head_idx, packet_success = 0;
  65         uint16_t avail_idx, res_cur_idx;
  66         uint16_t res_base_idx, res_end_idx;
  67         uint16_t free_entries;
  68         uint8_t success = 0;
  69
  70         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh);
  71         if (unlikely(queue_id != VIRTIO_RXQ)) {
  72                 LOG_DEBUG(VHOST_DATA, "mq isn't supported in this version.\n");
  73                 return 0;
  74         }
  75
  76         vq = dev->virtqueue[VIRTIO_RXQ];
  77         count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
  78
  79         /*
  80          * As many data cores may want access to available buffers,
  81          * they need to be reserved.
  82          */
  83         do {
  84                 res_base_idx = vq->last_used_idx_res;
  85                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
  86
  87                 free_entries = (avail_idx - res_base_idx);
  88                 /*check that we have enough buffers*/
  89                 if (unlikely(count > free_entries))
  90                         count = free_entries;
  91
  92                 if (count == 0)
  93                         return 0;
  94
  95                 res_end_idx = res_base_idx + count;
  96                 /* vq->last_used_idx_res is atomically updated. */
  97                 /* TODO: Allow to disable cmpset if no concurrency in application. */
  98                 success = rte_atomic16_cmpset(&vq->last_used_idx_res,
  99                                 res_base_idx, res_end_idx);
 100         } while (unlikely(success == 0));
 101         res_cur_idx = res_base_idx;
 102         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n",
 103                         dev->device_fh, res_cur_idx, res_end_idx);
 104
 105         /* Prefetch available ring to retrieve indexes. */
 106         rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]);
 107
 108         /* Retrieve all of the head indexes first to avoid caching issues. */
 109         for (head_idx = 0; head_idx < count; head_idx++)
 110                 head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) &
 111                                         (vq->size - 1)];
 112
 113         /*Prefetch descriptor index. */
 114         rte_prefetch0(&vq->desc[head[packet_success]]);
 115
 116         while (res_cur_idx != res_end_idx) {
 117                 uint32_t offset = 0, vb_offset = 0;
 118                 uint32_t pkt_len, len_to_cpy, data_len, total_copied = 0;
 119                 uint8_t hdr = 0, uncompleted_pkt = 0;
 120
 121                 /* Get descriptor from available ring */
 122                 desc = &vq->desc[head[packet_success]];
 123
 124                 buff = pkts[packet_success];
 125
 126                 /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
 127                 buff_addr = gpa_to_vva(dev, desc->addr);
 128                 /* Prefetch buffer address. */
 129                 rte_prefetch0((void *)(uintptr_t)buff_addr);
 130
 131                 /* Copy virtio_hdr to packet and increment buffer address */
 132                 buff_hdr_addr = buff_addr;
 133
 134                 /*
 135                  * If the descriptors are chained the header and data are
 136                  * placed in separate buffers.
 137                  */
 138                 if ((desc->flags & VRING_DESC_F_NEXT) &&
 139                         (desc->len == vq->vhost_hlen)) {
 140                         desc = &vq->desc[desc->next];
 141                         /* Buffer address translation. */
 142                         buff_addr = gpa_to_vva(dev, desc->addr);
 143                 } else {
 144                         vb_offset += vq->vhost_hlen;
 145                         hdr = 1;
 146                 }
 147
 148                 pkt_len = rte_pktmbuf_pkt_len(buff);
 149                 data_len = rte_pktmbuf_data_len(buff);
 150                 len_to_cpy = RTE_MIN(data_len,
 151                         hdr ? desc->len - vq->vhost_hlen : desc->len);
 152                 while (total_copied < pkt_len) {
 153                         /* Copy mbuf data to buffer */
 154                         rte_memcpy((void *)(uintptr_t)(buff_addr + vb_offset),
 155                                 (const void *)(rte_pktmbuf_mtod(buff, const char *) + offset),
 156                                 len_to_cpy);
 157                         PRINT_PACKET(dev, (uintptr_t)(buff_addr + vb_offset),
 158                                 len_to_cpy, 0);
 159
 160                         offset += len_to_cpy;
 161                         vb_offset += len_to_cpy;
 162                         total_copied += len_to_cpy;
 163
 164                         /* The whole packet completes */
 165                         if (total_copied == pkt_len)
 166                                 break;
 167
 168                         /* The current segment completes */
 169                         if (offset == data_len) {
 170                                 buff = buff->next;
 171                                 offset = 0;
 172                                 data_len = rte_pktmbuf_data_len(buff);
 173                         }
 174
 175                         /* The current vring descriptor done */
 176                         if (vb_offset == desc->len) {
 177                                 if (desc->flags & VRING_DESC_F_NEXT) {
 178                                         desc = &vq->desc[desc->next];
 179                                         buff_addr = gpa_to_vva(dev, desc->addr);
 180                                         vb_offset = 0;
 181                                 } else {
 182                                         /* Room in vring buffer is not enough */
 183                                         uncompleted_pkt = 1;
 184                                         break;
 185                                 }
 186                         }
 187                         len_to_cpy = RTE_MIN(data_len - offset, desc->len - vb_offset);
 188                 };
 189
 190                 /* Update used ring with desc information */
 191                 vq->used->ring[res_cur_idx & (vq->size - 1)].id =
 192                                                         head[packet_success];
 193
 194                 /* Drop the packet if it is uncompleted */
 195                 if (unlikely(uncompleted_pkt == 1))
 196                         vq->used->ring[res_cur_idx & (vq->size - 1)].len =
 197                                                         vq->vhost_hlen;
 198                 else
 199                         vq->used->ring[res_cur_idx & (vq->size - 1)].len =
 200                                                         pkt_len + vq->vhost_hlen;
 201
 202                 res_cur_idx++;
 203                 packet_success++;
 204
 205                 if (unlikely(uncompleted_pkt == 1))
 206                         continue;
 207
 208                 rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
 209                         (const void *)&virtio_hdr, vq->vhost_hlen);
 210
 211                 PRINT_PACKET(dev, (uintptr_t)buff_hdr_addr, vq->vhost_hlen, 1);
 212
 213                 if (res_cur_idx < res_end_idx) {
 214                         /* Prefetch descriptor index. */
 215                         rte_prefetch0(&vq->desc[head[packet_success]]);
 216                 }
 217         }
 218
 219         rte_compiler_barrier();
 220
 221         /* Wait until it's our turn to add our buffer to the used ring. */
 222         while (unlikely(vq->last_used_idx != res_base_idx))
 223                 rte_pause();
 224
 225         *(volatile uint16_t *)&vq->used->idx += count;
 226         vq->last_used_idx = res_end_idx;
 227
 228         /* flush used->idx update before we read avail->flags. */
 229         rte_mb();
 230
 231         /* Kick the guest if necessary. */
 232         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 233                 eventfd_write((int)vq->callfd, 1);
 234         return count;
 235 }
 236
 237 static inline uint32_t __attribute__((always_inline))
 238 copy_from_mbuf_to_vring(struct virtio_net *dev, uint16_t res_base_idx,
 239         uint16_t res_end_idx, struct rte_mbuf *pkt)
 240 {
 241         uint32_t vec_idx = 0;
 242         uint32_t entry_success = 0;
 243         struct vhost_virtqueue *vq;
 244         /* The virtio_hdr is initialised to 0. */
 245         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {
 246                 {0, 0, 0, 0, 0, 0}, 0};
 247         uint16_t cur_idx = res_base_idx;
 248         uint64_t vb_addr = 0;
 249         uint64_t vb_hdr_addr = 0;
 250         uint32_t seg_offset = 0;
 251         uint32_t vb_offset = 0;
 252         uint32_t seg_avail;
 253         uint32_t vb_avail;
 254         uint32_t cpy_len, entry_len;
 255
 256         if (pkt == NULL)
 257                 return 0;
 258
 259         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| "
 260                 "End Index %d\n",
 261                 dev->device_fh, cur_idx, res_end_idx);
 262
 263         /*
 264          * Convert from gpa to vva
 265          * (guest physical addr -> vhost virtual addr)
 266          */
 267         vq = dev->virtqueue[VIRTIO_RXQ];
 268         vb_addr =
 269                 gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 270         vb_hdr_addr = vb_addr;
 271
 272         /* Prefetch buffer address. */
 273         rte_prefetch0((void *)(uintptr_t)vb_addr);
 274
 275         virtio_hdr.num_buffers = res_end_idx - res_base_idx;
 276
 277         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") RX: Num merge buffers %d\n",
 278                 dev->device_fh, virtio_hdr.num_buffers);
 279
 280         rte_memcpy((void *)(uintptr_t)vb_hdr_addr,
 281                 (const void *)&virtio_hdr, vq->vhost_hlen);
 282
 283         PRINT_PACKET(dev, (uintptr_t)vb_hdr_addr, vq->vhost_hlen, 1);
 284
 285         seg_avail = rte_pktmbuf_data_len(pkt);
 286         vb_offset = vq->vhost_hlen;
 287         vb_avail =
 288                 vq->buf_vec[vec_idx].buf_len - vq->vhost_hlen;
 289
 290         entry_len = vq->vhost_hlen;
 291
 292         if (vb_avail == 0) {
 293                 uint32_t desc_idx =
 294                         vq->buf_vec[vec_idx].desc_idx;
 295                 vq->desc[desc_idx].len = vq->vhost_hlen;
 296
 297                 if ((vq->desc[desc_idx].flags
 298                         & VRING_DESC_F_NEXT) == 0) {
 299                         /* Update used ring with desc information */
 300                         vq->used->ring[cur_idx & (vq->size - 1)].id
 301                                 = vq->buf_vec[vec_idx].desc_idx;
 302                         vq->used->ring[cur_idx & (vq->size - 1)].len
 303                                 = entry_len;
 304
 305                         entry_len = 0;
 306                         cur_idx++;
 307                         entry_success++;
 308                 }
 309
 310                 vec_idx++;
 311                 vb_addr =
 312                         gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 313
 314                 /* Prefetch buffer address. */
 315                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 316                 vb_offset = 0;
 317                 vb_avail = vq->buf_vec[vec_idx].buf_len;
 318         }
 319
 320         cpy_len = RTE_MIN(vb_avail, seg_avail);
 321
 322         while (cpy_len > 0) {
 323                 /* Copy mbuf data to vring buffer */
 324                 rte_memcpy((void *)(uintptr_t)(vb_addr + vb_offset),
 325                         (const void *)(rte_pktmbuf_mtod(pkt, char*) + seg_offset),
 326                         cpy_len);
 327
 328                 PRINT_PACKET(dev,
 329                         (uintptr_t)(vb_addr + vb_offset),
 330                         cpy_len, 0);
 331
 332                 seg_offset += cpy_len;
 333                 vb_offset += cpy_len;
 334                 seg_avail -= cpy_len;
 335                 vb_avail -= cpy_len;
 336                 entry_len += cpy_len;
 337
 338                 if (seg_avail != 0) {
 339                         /*
 340                          * The virtio buffer in this vring
 341                          * entry reach to its end.
 342                          * But the segment doesn't complete.
 343                          */
 344                         if ((vq->desc[vq->buf_vec[vec_idx].desc_idx].flags &
 345                                 VRING_DESC_F_NEXT) == 0) {
 346                                 /* Update used ring with desc information */
 347                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 348                                         = vq->buf_vec[vec_idx].desc_idx;
 349                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 350                                         = entry_len;
 351                                 entry_len = 0;
 352                                 cur_idx++;
 353                                 entry_success++;
 354                         }
 355
 356                         vec_idx++;
 357                         vb_addr = gpa_to_vva(dev,
 358                                 vq->buf_vec[vec_idx].buf_addr);
 359                         vb_offset = 0;
 360                         vb_avail = vq->buf_vec[vec_idx].buf_len;
 361                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 362                 } else {
 363                         /*
 364                          * This current segment complete, need continue to
 365                          * check if the whole packet complete or not.
 366                          */
 367                         pkt = pkt->next;
 368                         if (pkt != NULL) {
 369                                 /*
 370                                  * There are more segments.
 371                                  */
 372                                 if (vb_avail == 0) {
 373                                         /*
 374                                          * This current buffer from vring is
 375                                          * used up, need fetch next buffer
 376                                          * from buf_vec.
 377                                          */
 378                                         uint32_t desc_idx =
 379                                                 vq->buf_vec[vec_idx].desc_idx;
 380                                         vq->desc[desc_idx].len = vb_offset;
 381
 382                                         if ((vq->desc[desc_idx].flags &
 383                                                 VRING_DESC_F_NEXT) == 0) {
 384                                                 uint16_t wrapped_idx =
 385                                                         cur_idx & (vq->size - 1);
 386                                                 /*
 387                                                  * Update used ring with the
 388                                                  * descriptor information
 389                                                  */
 390                                                 vq->used->ring[wrapped_idx].id
 391                                                         = desc_idx;
 392                                                 vq->used->ring[wrapped_idx].len
 393                                                         = entry_len;
 394                                                 entry_success++;
 395                                                 entry_len = 0;
 396                                                 cur_idx++;
 397                                         }
 398
 399                                         /* Get next buffer from buf_vec. */
 400                                         vec_idx++;
 401                                         vb_addr = gpa_to_vva(dev,
 402                                                 vq->buf_vec[vec_idx].buf_addr);
 403                                         vb_avail =
 404                                                 vq->buf_vec[vec_idx].buf_len;
 405                                         vb_offset = 0;
 406                                 }
 407
 408                                 seg_offset = 0;
 409                                 seg_avail = rte_pktmbuf_data_len(pkt);
 410                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 411                         } else {
 412                                 /*
 413                                  * This whole packet completes.
 414                                  */
 415                                 uint32_t desc_idx =
 416                                         vq->buf_vec[vec_idx].desc_idx;
 417                                 vq->desc[desc_idx].len = vb_offset;
 418
 419                                 while (vq->desc[desc_idx].flags &
 420                                         VRING_DESC_F_NEXT) {
 421                                         desc_idx = vq->desc[desc_idx].next;
 422                                          vq->desc[desc_idx].len = 0;
 423                                 }
 424
 425                                 /* Update used ring with desc information */
 426                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 427                                         = vq->buf_vec[vec_idx].desc_idx;
 428                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 429                                         = entry_len;
 430                                 entry_len = 0;
 431                                 cur_idx++;
 432                                 entry_success++;
 433                                 seg_avail = 0;
 434                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 435                         }
 436                 }
 437         }
 438
 439         return entry_success;
 440 }
 441
 442 /*
 443  * This function works for mergeable RX.
 444  */
 445 static inline uint32_t __attribute__((always_inline))
 446 virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 447         struct rte_mbuf **pkts, uint32_t count)
 448 {
 449         struct vhost_virtqueue *vq;
 450         uint32_t pkt_idx = 0, entry_success = 0;
 451         uint16_t avail_idx, res_cur_idx;
 452         uint16_t res_base_idx, res_end_idx;
 453         uint8_t success = 0;
 454
 455         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_rx()\n",
 456                 dev->device_fh);
 457         if (unlikely(queue_id != VIRTIO_RXQ)) {
 458                 LOG_DEBUG(VHOST_DATA, "mq isn't supported in this version.\n");
 459         }
 460
 461         vq = dev->virtqueue[VIRTIO_RXQ];
 462         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
 463
 464         if (count == 0)
 465                 return 0;
 466
 467         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 468                 uint32_t secure_len = 0;
 469                 uint16_t need_cnt;
 470                 uint32_t vec_idx = 0;
 471                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vhost_hlen;
 472                 uint16_t i, id;
 473
 474                 do {
 475                         /*
 476                          * As many data cores may want access to available
 477                          * buffers, they need to be reserved.
 478                          */
 479                         res_base_idx = vq->last_used_idx_res;
 480                         res_cur_idx = res_base_idx;
 481
 482                         do {
 483                                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 484                                 if (unlikely(res_cur_idx == avail_idx)) {
 485                                         LOG_DEBUG(VHOST_DATA,
 486                                                 "(%"PRIu64") Failed "
 487                                                 "to get enough desc from "
 488                                                 "vring\n",
 489                                                 dev->device_fh);
 490                                         return pkt_idx;
 491                                 } else {
 492                                         uint16_t wrapped_idx =
 493                                                 (res_cur_idx) & (vq->size - 1);
 494                                         uint32_t idx =
 495                                                 vq->avail->ring[wrapped_idx];
 496                                         uint8_t next_desc;
 497
 498                                         do {
 499                                                 next_desc = 0;
 500                                                 secure_len += vq->desc[idx].len;
 501                                                 if (vq->desc[idx].flags &
 502                                                         VRING_DESC_F_NEXT) {
 503                                                         idx = vq->desc[idx].next;
 504                                                         next_desc = 1;
 505                                                 }
 506                                         } while (next_desc);
 507
 508                                         res_cur_idx++;
 509                                 }
 510                         } while (pkt_len > secure_len);
 511
 512                         /* vq->last_used_idx_res is atomically updated. */
 513                         success = rte_atomic16_cmpset(&vq->last_used_idx_res,
 514                                                         res_base_idx,
 515                                                         res_cur_idx);
 516                 } while (success == 0);
 517
 518                 id = res_base_idx;
 519                 need_cnt = res_cur_idx - res_base_idx;
 520
 521                 for (i = 0; i < need_cnt; i++, id++) {
 522                         uint16_t wrapped_idx = id & (vq->size - 1);
 523                         uint32_t idx = vq->avail->ring[wrapped_idx];
 524                         uint8_t next_desc;
 525                         do {
 526                                 next_desc = 0;
 527                                 vq->buf_vec[vec_idx].buf_addr =
 528                                         vq->desc[idx].addr;
 529                                 vq->buf_vec[vec_idx].buf_len =
 530                                         vq->desc[idx].len;
 531                                 vq->buf_vec[vec_idx].desc_idx = idx;
 532                                 vec_idx++;
 533
 534                                 if (vq->desc[idx].flags & VRING_DESC_F_NEXT) {
 535                                         idx = vq->desc[idx].next;
 536                                         next_desc = 1;
 537                                 }
 538                         } while (next_desc);
 539                 }
 540
 541                 res_end_idx = res_cur_idx;
 542
 543                 entry_success = copy_from_mbuf_to_vring(dev, res_base_idx,
 544                         res_end_idx, pkts[pkt_idx]);
 545
 546                 rte_compiler_barrier();
 547
 548                 /*
 549                  * Wait until it's our turn to add our buffer
 550                  * to the used ring.
 551                  */
 552                 while (unlikely(vq->last_used_idx != res_base_idx))
 553                         rte_pause();
 554
 555                 *(volatile uint16_t *)&vq->used->idx += entry_success;
 556                 vq->last_used_idx = res_end_idx;
 557
 558                 /* flush used->idx update before we read avail->flags. */
 559                 rte_mb();
 560
 561                 /* Kick the guest if necessary. */
 562                 if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 563                         eventfd_write((int)vq->callfd, 1);
 564         }
 565
 566         return count;
 567 }
 568
 569 uint16_t
 570 rte_vhost_enqueue_burst(struct virtio_net *dev, uint16_t queue_id,
 571         struct rte_mbuf **pkts, uint16_t count)
 572 {
 573         if (unlikely(dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)))
 574                 return virtio_dev_merge_rx(dev, queue_id, pkts, count);
 575         else
 576                 return virtio_dev_rx(dev, queue_id, pkts, count);
 577 }
 578
 579 uint16_t
 580 rte_vhost_dequeue_burst(struct virtio_net *dev, uint16_t queue_id,
 581         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 582 {
 583         struct rte_mbuf *m, *prev;
 584         struct vhost_virtqueue *vq;
 585         struct vring_desc *desc;
 586         uint64_t vb_addr = 0;
 587         uint32_t head[MAX_PKT_BURST];
 588         uint32_t used_idx;
 589         uint32_t i;
 590         uint16_t free_entries, entry_success = 0;
 591         uint16_t avail_idx;
 592
 593         if (unlikely(queue_id != VIRTIO_TXQ)) {
 594                 LOG_DEBUG(VHOST_DATA, "mq isn't supported in this version.\n");
 595                 return 0;
 596         }
 597
 598         vq = dev->virtqueue[VIRTIO_TXQ];
 599         avail_idx =  *((volatile uint16_t *)&vq->avail->idx);
 600
 601         /* If there are no available buffers then return. */
 602         if (vq->last_used_idx == avail_idx)
 603                 return 0;
 604
 605         LOG_DEBUG(VHOST_DATA, "%s (%"PRIu64")\n", __func__,
 606                 dev->device_fh);
 607
 608         /* Prefetch available ring to retrieve head indexes. */
 609         rte_prefetch0(&vq->avail->ring[vq->last_used_idx & (vq->size - 1)]);
 610
 611         /*get the number of free entries in the ring*/
 612         free_entries = (avail_idx - vq->last_used_idx);
 613
 614         free_entries = RTE_MIN(free_entries, count);
 615         /* Limit to MAX_PKT_BURST. */
 616         free_entries = RTE_MIN(free_entries, MAX_PKT_BURST);
 617
 618         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n",
 619                         dev->device_fh, free_entries);
 620         /* Retrieve all of the head indexes first to avoid caching issues. */
 621         for (i = 0; i < free_entries; i++)
 622                 head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)];
 623
 624         /* Prefetch descriptor index. */
 625         rte_prefetch0(&vq->desc[head[entry_success]]);
 626         rte_prefetch0(&vq->used->ring[vq->last_used_idx & (vq->size - 1)]);
 627
 628         while (entry_success < free_entries) {
 629                 uint32_t vb_avail, vb_offset;
 630                 uint32_t seg_avail, seg_offset;
 631                 uint32_t cpy_len;
 632                 uint32_t seg_num = 0;
 633                 struct rte_mbuf *cur;
 634                 uint8_t alloc_err = 0;
 635
 636                 desc = &vq->desc[head[entry_success]];
 637
 638                 /* Discard first buffer as it is the virtio header */
 639                 if (desc->flags & VRING_DESC_F_NEXT) {
 640                         desc = &vq->desc[desc->next];
 641                         vb_offset = 0;
 642                         vb_avail = desc->len;
 643                 } else {
 644                         vb_offset = vq->vhost_hlen;
 645                         vb_avail = desc->len - vb_offset;
 646                 }
 647
 648                 /* Buffer address translation. */
 649                 vb_addr = gpa_to_vva(dev, desc->addr);
 650                 /* Prefetch buffer address. */
 651                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 652
 653                 used_idx = vq->last_used_idx & (vq->size - 1);
 654
 655                 if (entry_success < (free_entries - 1)) {
 656                         /* Prefetch descriptor index. */
 657                         rte_prefetch0(&vq->desc[head[entry_success+1]]);
 658                         rte_prefetch0(&vq->used->ring[(used_idx + 1) & (vq->size - 1)]);
 659                 }
 660
 661                 /* Update used index buffer information. */
 662                 vq->used->ring[used_idx].id = head[entry_success];
 663                 vq->used->ring[used_idx].len = 0;
 664
 665                 /* Allocate an mbuf and populate the structure. */
 666                 m = rte_pktmbuf_alloc(mbuf_pool);
 667                 if (unlikely(m == NULL)) {
 668                         RTE_LOG(ERR, VHOST_DATA,
 669                                 "Failed to allocate memory for mbuf.\n");
 670                         break;
 671                 }
 672                 seg_offset = 0;
 673                 seg_avail = m->buf_len - RTE_PKTMBUF_HEADROOM;
 674                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 675
 676                 PRINT_PACKET(dev, (uintptr_t)vb_addr, desc->len, 0);
 677
 678                 seg_num++;
 679                 cur = m;
 680                 prev = m;
 681                 while (cpy_len != 0) {
 682                         rte_memcpy((void *)(rte_pktmbuf_mtod(cur, char *) + seg_offset),
 683                                 (void *)((uintptr_t)(vb_addr + vb_offset)),
 684                                 cpy_len);
 685
 686                         seg_offset += cpy_len;
 687                         vb_offset += cpy_len;
 688                         vb_avail -= cpy_len;
 689                         seg_avail -= cpy_len;
 690
 691                         if (vb_avail != 0) {
 692                                 /*
 693                                  * The segment reachs to its end,
 694                                  * while the virtio buffer in TX vring has
 695                                  * more data to be copied.
 696                                  */
 697                                 cur->data_len = seg_offset;
 698                                 m->pkt_len += seg_offset;
 699                                 /* Allocate mbuf and populate the structure. */
 700                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 701                                 if (unlikely(cur == NULL)) {
 702                                         RTE_LOG(ERR, VHOST_DATA, "Failed to "
 703                                                 "allocate memory for mbuf.\n");
 704                                         rte_pktmbuf_free(m);
 705                                         alloc_err = 1;
 706                                         break;
 707                                 }
 708
 709                                 seg_num++;
 710                                 prev->next = cur;
 711                                 prev = cur;
 712                                 seg_offset = 0;
 713                                 seg_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM;
 714                         } else {
 715                                 if (desc->flags & VRING_DESC_F_NEXT) {
 716                                         /*
 717                                          * There are more virtio buffers in
 718                                          * same vring entry need to be copied.
 719                                          */
 720                                         if (seg_avail == 0) {
 721                                                 /*
 722                                                  * The current segment hasn't
 723                                                  * room to accomodate more
 724                                                  * data.
 725                                                  */
 726                                                 cur->data_len = seg_offset;
 727                                                 m->pkt_len += seg_offset;
 728                                                 /*
 729                                                  * Allocate an mbuf and
 730                                                  * populate the structure.
 731                                                  */
 732                                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 733                                                 if (unlikely(cur == NULL)) {
 734                                                         RTE_LOG(ERR,
 735                                                                 VHOST_DATA,
 736                                                                 "Failed to "
 737                                                                 "allocate memory "
 738                                                                 "for mbuf\n");
 739                                                         rte_pktmbuf_free(m);
 740                                                         alloc_err = 1;
 741                                                         break;
 742                                                 }
 743                                                 seg_num++;
 744                                                 prev->next = cur;
 745                                                 prev = cur;
 746                                                 seg_offset = 0;
 747                                                 seg_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM;
 748                                         }
 749
 750                                         desc = &vq->desc[desc->next];
 751
 752                                         /* Buffer address translation. */
 753                                         vb_addr = gpa_to_vva(dev, desc->addr);
 754                                         /* Prefetch buffer address. */
 755                                         rte_prefetch0((void *)(uintptr_t)vb_addr);
 756                                         vb_offset = 0;
 757                                         vb_avail = desc->len;
 758
 759                                         PRINT_PACKET(dev, (uintptr_t)vb_addr,
 760                                                 desc->len, 0);
 761                                 } else {
 762                                         /* The whole packet completes. */
 763                                         cur->data_len = seg_offset;
 764                                         m->pkt_len += seg_offset;
 765                                         vb_avail = 0;
 766                                 }
 767                         }
 768
 769                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 770                 }
 771
 772                 if (unlikely(alloc_err == 1))
 773                         break;
 774
 775                 m->nb_segs = seg_num;
 776
 777                 pkts[entry_success] = m;
 778                 vq->last_used_idx++;
 779                 entry_success++;
 780         }
 781
 782         rte_compiler_barrier();
 783         vq->used->idx += entry_success;
 784         /* Kick guest if required. */
 785         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 786                 eventfd_write((int)vq->callfd, 1);
 787         return entry_success;
 788 }