lib/librte_vhost/vhost_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <arpa/inet.h>
  35 #include <getopt.h>
  36 #include <linux/if_ether.h>
  37 #include <linux/if_vlan.h>
  38 #include <linux/virtio_net.h>
  39 #include <linux/virtio_ring.h>
  40 #include <signal.h>
  41 #include <stdint.h>
  42 #include <sys/eventfd.h>
  43 #include <sys/param.h>
  44 #include <unistd.h>
  45
  46 #include <rte_atomic.h>
  47 #include <rte_cycles.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_log.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_malloc.h>
  52
  53 #include "main.h"
  54 #include "virtio-net.h"
  55 #include "vhost-net-cdev.h"
  56
  57 #define MAX_PKT_BURST 32                /* Max burst size for RX/TX */
  58
  59 /*
  60  * Function to convert guest physical addresses to vhost virtual addresses. This
  61  * is used to convert virtio buffer addresses.
  62  */
  63 static inline uint64_t __attribute__((always_inline))
  64 gpa_to_vva(struct virtio_net *dev, uint64_t guest_pa)
  65 {
  66         struct virtio_memory_regions *region;
  67         uint32_t regionidx;
  68         uint64_t vhost_va = 0;
  69
  70         for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
  71                 region = &dev->mem->regions[regionidx];
  72                 if ((guest_pa >= region->guest_phys_address) &&
  73                         (guest_pa <= region->guest_phys_address_end)) {
  74                         vhost_va = region->address_offset + guest_pa;
  75                         break;
  76                 }
  77         }
  78         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") GPA %p| VVA %p\n",
  79                 dev->device_fh, (void*)(uintptr_t)guest_pa, (void*)(uintptr_t)vhost_va);
  80
  81         return vhost_va;
  82 }
  83
  84 /*
  85  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
  86  * be received from the physical port or from another virtio device. A packet
  87  * count is returned to indicate the number of packets that were succesfully
  88  * added to the RX queue. This function works when mergeable is disabled.
  89  */
  90 static inline uint32_t __attribute__((always_inline))
  91 virtio_dev_rx(struct virtio_net *dev, struct rte_mbuf **pkts, uint32_t count)
  92 {
  93         struct vhost_virtqueue *vq;
  94         struct vring_desc *desc;
  95         struct rte_mbuf *buff;
  96         /* The virtio_hdr is initialised to 0. */
  97         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0,0,0,0,0,0},0};
  98         uint64_t buff_addr = 0;
  99         uint64_t buff_hdr_addr = 0;
 100         uint32_t head[MAX_PKT_BURST], packet_len = 0;
 101         uint32_t head_idx, packet_success = 0;
 102         uint16_t avail_idx, res_cur_idx;
 103         uint16_t res_base_idx, res_end_idx;
 104         uint16_t free_entries;
 105         uint8_t success = 0;
 106
 107         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh);
 108         vq = dev->virtqueue[VIRTIO_RXQ];
 109         count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
 110
 111         /* As many data cores may want access to available buffers, they need to be reserved. */
 112         do {
 113                 res_base_idx = vq->last_used_idx_res;
 114                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 115
 116                 free_entries = (avail_idx - res_base_idx);
 117                 /*check that we have enough buffers*/
 118                 if (unlikely(count > free_entries))
 119                         count = free_entries;
 120
 121                 if (count == 0)
 122                         return 0;
 123
 124                 res_end_idx = res_base_idx + count;
 125                 /* vq->last_used_idx_res is atomically updated. */
 126                 success = rte_atomic16_cmpset(&vq->last_used_idx_res, res_base_idx,
 127                                                                         res_end_idx);
 128         } while (unlikely(success == 0));
 129         res_cur_idx = res_base_idx;
 130         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n", dev->device_fh, res_cur_idx, res_end_idx);
 131
 132         /* Prefetch available ring to retrieve indexes. */
 133         rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]);
 134
 135         /* Retrieve all of the head indexes first to avoid caching issues. */
 136         for (head_idx = 0; head_idx < count; head_idx++)
 137                 head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & (vq->size - 1)];
 138
 139         /*Prefetch descriptor index. */
 140         rte_prefetch0(&vq->desc[head[packet_success]]);
 141
 142         while (res_cur_idx != res_end_idx) {
 143                 /* Get descriptor from available ring */
 144                 desc = &vq->desc[head[packet_success]];
 145
 146                 buff = pkts[packet_success];
 147
 148                 /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
 149                 buff_addr = gpa_to_vva(dev, desc->addr);
 150                 /* Prefetch buffer address. */
 151                 rte_prefetch0((void*)(uintptr_t)buff_addr);
 152
 153                 /* Copy virtio_hdr to packet and increment buffer address */
 154                 buff_hdr_addr = buff_addr;
 155                 packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen;
 156
 157                 /*
 158                  * If the descriptors are chained the header and data are
 159                  * placed in separate buffers.
 160                  */
 161                 if (desc->flags & VRING_DESC_F_NEXT) {
 162                         desc->len = vq->vhost_hlen;
 163                         desc = &vq->desc[desc->next];
 164                         /* Buffer address translation. */
 165                         buff_addr = gpa_to_vva(dev, desc->addr);
 166                         desc->len = rte_pktmbuf_data_len(buff);
 167                 } else {
 168                         buff_addr += vq->vhost_hlen;
 169                         desc->len = packet_len;
 170                 }
 171
 172                 /* Update used ring with desc information */
 173                 vq->used->ring[res_cur_idx & (vq->size - 1)].id = head[packet_success];
 174                 vq->used->ring[res_cur_idx & (vq->size - 1)].len = packet_len;
 175
 176                 /* Copy mbuf data to buffer */
 177                 rte_memcpy((void *)(uintptr_t)buff_addr,
 178                         rte_pktmbuf_mtod(buff, const void *),
 179                         rte_pktmbuf_data_len(buff));
 180                 PRINT_PACKET(dev, (uintptr_t)buff_addr,
 181                         rte_pktmbuf_data_len(buff), 0);
 182
 183                 res_cur_idx++;
 184                 packet_success++;
 185
 186                 rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
 187                         (const void *)&virtio_hdr, vq->vhost_hlen);
 188
 189                 PRINT_PACKET(dev, (uintptr_t)buff_hdr_addr, vq->vhost_hlen, 1);
 190
 191                 if (res_cur_idx < res_end_idx) {
 192                         /* Prefetch descriptor index. */
 193                         rte_prefetch0(&vq->desc[head[packet_success]]);
 194                 }
 195         }
 196
 197         rte_compiler_barrier();
 198
 199         /* Wait until it's our turn to add our buffer to the used ring. */
 200         while (unlikely(vq->last_used_idx != res_base_idx))
 201                 rte_pause();
 202
 203         *(volatile uint16_t *)&vq->used->idx += count;
 204         vq->last_used_idx = res_end_idx;
 205
 206         /* Kick the guest if necessary. */
 207         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 208                 eventfd_write((int)vq->kickfd, 1);
 209         return count;
 210 }
 211
 212 static inline uint32_t __attribute__((always_inline))
 213 copy_from_mbuf_to_vring(struct virtio_net *dev,
 214         uint16_t res_base_idx, uint16_t res_end_idx,
 215         struct rte_mbuf *pkt)
 216 {
 217         uint32_t vec_idx = 0;
 218         uint32_t entry_success = 0;
 219         struct vhost_virtqueue *vq;
 220         /* The virtio_hdr is initialised to 0. */
 221         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {
 222                 {0, 0, 0, 0, 0, 0}, 0};
 223         uint16_t cur_idx = res_base_idx;
 224         uint64_t vb_addr = 0;
 225         uint64_t vb_hdr_addr = 0;
 226         uint32_t seg_offset = 0;
 227         uint32_t vb_offset = 0;
 228         uint32_t seg_avail;
 229         uint32_t vb_avail;
 230         uint32_t cpy_len, entry_len;
 231
 232         if (pkt == NULL)
 233                 return 0;
 234
 235         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| "
 236                 "End Index %d\n",
 237                 dev->device_fh, cur_idx, res_end_idx);
 238
 239         /*
 240          * Convert from gpa to vva
 241          * (guest physical addr -> vhost virtual addr)
 242          */
 243         vq = dev->virtqueue[VIRTIO_RXQ];
 244         vb_addr =
 245                 gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 246         vb_hdr_addr = vb_addr;
 247
 248         /* Prefetch buffer address. */
 249         rte_prefetch0((void *)(uintptr_t)vb_addr);
 250
 251         virtio_hdr.num_buffers = res_end_idx - res_base_idx;
 252
 253         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") RX: Num merge buffers %d\n",
 254                 dev->device_fh, virtio_hdr.num_buffers);
 255
 256         rte_memcpy((void *)(uintptr_t)vb_hdr_addr,
 257                 (const void *)&virtio_hdr, vq->vhost_hlen);
 258
 259         PRINT_PACKET(dev, (uintptr_t)vb_hdr_addr, vq->vhost_hlen, 1);
 260
 261         seg_avail = rte_pktmbuf_data_len(pkt);
 262         vb_offset = vq->vhost_hlen;
 263         vb_avail =
 264                 vq->buf_vec[vec_idx].buf_len - vq->vhost_hlen;
 265
 266         entry_len = vq->vhost_hlen;
 267
 268         if (vb_avail == 0) {
 269                 uint32_t desc_idx =
 270                         vq->buf_vec[vec_idx].desc_idx;
 271                 vq->desc[desc_idx].len = vq->vhost_hlen;
 272
 273                 if ((vq->desc[desc_idx].flags
 274                         & VRING_DESC_F_NEXT) == 0) {
 275                         /* Update used ring with desc information */
 276                         vq->used->ring[cur_idx & (vq->size - 1)].id
 277                                 = vq->buf_vec[vec_idx].desc_idx;
 278                         vq->used->ring[cur_idx & (vq->size - 1)].len
 279                                 = entry_len;
 280
 281                         entry_len = 0;
 282                         cur_idx++;
 283                         entry_success++;
 284                 }
 285
 286                 vec_idx++;
 287                 vb_addr =
 288                         gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 289
 290                 /* Prefetch buffer address. */
 291                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 292                 vb_offset = 0;
 293                 vb_avail = vq->buf_vec[vec_idx].buf_len;
 294         }
 295
 296         cpy_len = RTE_MIN(vb_avail, seg_avail);
 297
 298         while (cpy_len > 0) {
 299                 /* Copy mbuf data to vring buffer */
 300                 rte_memcpy((void *)(uintptr_t)(vb_addr + vb_offset),
 301                         (const void *)(rte_pktmbuf_mtod(pkt, char*) + seg_offset),
 302                         cpy_len);
 303
 304                 PRINT_PACKET(dev,
 305                         (uintptr_t)(vb_addr + vb_offset),
 306                         cpy_len, 0);
 307
 308                 seg_offset += cpy_len;
 309                 vb_offset += cpy_len;
 310                 seg_avail -= cpy_len;
 311                 vb_avail -= cpy_len;
 312                 entry_len += cpy_len;
 313
 314                 if (seg_avail != 0) {
 315                         /*
 316                          * The virtio buffer in this vring
 317                          * entry reach to its end.
 318                          * But the segment doesn't complete.
 319                          */
 320                         if ((vq->desc[vq->buf_vec[vec_idx].desc_idx].flags &
 321                                 VRING_DESC_F_NEXT) == 0) {
 322                                 /* Update used ring with desc information */
 323                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 324                                         = vq->buf_vec[vec_idx].desc_idx;
 325                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 326                                         = entry_len;
 327                                 entry_len = 0;
 328                                 cur_idx++;
 329                                 entry_success++;
 330                         }
 331
 332                         vec_idx++;
 333                         vb_addr = gpa_to_vva(dev,
 334                                 vq->buf_vec[vec_idx].buf_addr);
 335                         vb_offset = 0;
 336                         vb_avail = vq->buf_vec[vec_idx].buf_len;
 337                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 338                 } else {
 339                         /*
 340                          * This current segment complete, need continue to
 341                          * check if the whole packet complete or not.
 342                          */
 343                         pkt = pkt->next;
 344                         if (pkt != NULL) {
 345                                 /*
 346                                  * There are more segments.
 347                                  */
 348                                 if (vb_avail == 0) {
 349                                         /*
 350                                          * This current buffer from vring is
 351                                          * used up, need fetch next buffer
 352                                          * from buf_vec.
 353                                          */
 354                                         uint32_t desc_idx =
 355                                                 vq->buf_vec[vec_idx].desc_idx;
 356                                         vq->desc[desc_idx].len = vb_offset;
 357
 358                                         if ((vq->desc[desc_idx].flags &
 359                                                 VRING_DESC_F_NEXT) == 0) {
 360                                                 uint16_t wrapped_idx =
 361                                                         cur_idx & (vq->size - 1);
 362                                                 /*
 363                                                  * Update used ring with the
 364                                                  * descriptor information
 365                                                  */
 366                                                 vq->used->ring[wrapped_idx].id
 367                                                         = desc_idx;
 368                                                 vq->used->ring[wrapped_idx].len
 369                                                         = entry_len;
 370                                                 entry_success++;
 371                                                 entry_len = 0;
 372                                                 cur_idx++;
 373                                         }
 374
 375                                         /* Get next buffer from buf_vec. */
 376                                         vec_idx++;
 377                                         vb_addr = gpa_to_vva(dev,
 378                                                 vq->buf_vec[vec_idx].buf_addr);
 379                                         vb_avail =
 380                                                 vq->buf_vec[vec_idx].buf_len;
 381                                         vb_offset = 0;
 382                                 }
 383
 384                                 seg_offset = 0;
 385                                 seg_avail = rte_pktmbuf_data_len(pkt);
 386                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 387                         } else {
 388                                 /*
 389                                  * This whole packet completes.
 390                                  */
 391                                 uint32_t desc_idx =
 392                                         vq->buf_vec[vec_idx].desc_idx;
 393                                 vq->desc[desc_idx].len = vb_offset;
 394
 395                                 while (vq->desc[desc_idx].flags &
 396                                         VRING_DESC_F_NEXT) {
 397                                         desc_idx = vq->desc[desc_idx].next;
 398                                          vq->desc[desc_idx].len = 0;
 399                                 }
 400
 401                                 /* Update used ring with desc information */
 402                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 403                                         = vq->buf_vec[vec_idx].desc_idx;
 404                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 405                                         = entry_len;
 406                                 entry_len = 0;
 407                                 cur_idx++;
 408                                 entry_success++;
 409                                 seg_avail = 0;
 410                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 411                         }
 412                 }
 413         }
 414
 415         return entry_success;
 416 }
 417
 418 /*
 419  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
 420  * be received from the physical port or from another virtio device. A packet
 421  * count is returned to indicate the number of packets that were succesfully
 422  * added to the RX queue. This function works for mergeable RX.
 423  */
 424 static inline uint32_t __attribute__((always_inline))
 425 virtio_dev_merge_rx(struct virtio_net *dev, struct rte_mbuf **pkts,
 426         uint32_t count)
 427 {
 428         struct vhost_virtqueue *vq;
 429         uint32_t pkt_idx = 0, entry_success = 0;
 430         uint16_t avail_idx, res_cur_idx;
 431         uint16_t res_base_idx, res_end_idx;
 432         uint8_t success = 0;
 433
 434         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_rx()\n",
 435                 dev->device_fh);
 436         vq = dev->virtqueue[VIRTIO_RXQ];
 437         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
 438
 439         if (count == 0)
 440                 return 0;
 441
 442         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 443                 uint32_t secure_len = 0;
 444                 uint16_t need_cnt;
 445                 uint32_t vec_idx = 0;
 446                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vhost_hlen;
 447                 uint16_t i, id;
 448
 449                 do {
 450                         /*
 451                          * As many data cores may want access to available
 452                          * buffers, they need to be reserved.
 453                          */
 454                         res_base_idx = vq->last_used_idx_res;
 455                         res_cur_idx = res_base_idx;
 456
 457                         do {
 458                                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 459                                 if (unlikely(res_cur_idx == avail_idx)) {
 460                                         LOG_DEBUG(VHOST_DATA,
 461                                                 "(%"PRIu64") Failed "
 462                                                 "to get enough desc from "
 463                                                 "vring\n",
 464                                                 dev->device_fh);
 465                                         return pkt_idx;
 466                                 } else {
 467                                         uint16_t wrapped_idx =
 468                                                 (res_cur_idx) & (vq->size - 1);
 469                                         uint32_t idx =
 470                                                 vq->avail->ring[wrapped_idx];
 471                                         uint8_t next_desc;
 472
 473                                         do {
 474                                                 next_desc = 0;
 475                                                 secure_len += vq->desc[idx].len;
 476                                                 if (vq->desc[idx].flags &
 477                                                         VRING_DESC_F_NEXT) {
 478                                                         idx = vq->desc[idx].next;
 479                                                         next_desc = 1;
 480                                                 }
 481                                         } while (next_desc);
 482
 483                                         res_cur_idx++;
 484                                 }
 485                         } while (pkt_len > secure_len);
 486
 487                         /* vq->last_used_idx_res is atomically updated. */
 488                         success = rte_atomic16_cmpset(&vq->last_used_idx_res,
 489                                                         res_base_idx,
 490                                                         res_cur_idx);
 491                 } while (success == 0);
 492
 493                 id = res_base_idx;
 494                 need_cnt = res_cur_idx - res_base_idx;
 495
 496                 for (i = 0; i < need_cnt; i++, id++) {
 497                         uint16_t wrapped_idx = id & (vq->size - 1);
 498                         uint32_t idx = vq->avail->ring[wrapped_idx];
 499                         uint8_t next_desc;
 500                         do {
 501                                 next_desc = 0;
 502                                 vq->buf_vec[vec_idx].buf_addr =
 503                                         vq->desc[idx].addr;
 504                                 vq->buf_vec[vec_idx].buf_len =
 505                                         vq->desc[idx].len;
 506                                 vq->buf_vec[vec_idx].desc_idx = idx;
 507                                 vec_idx++;
 508
 509                                 if (vq->desc[idx].flags & VRING_DESC_F_NEXT) {
 510                                         idx = vq->desc[idx].next;
 511                                         next_desc = 1;
 512                                 }
 513                         } while (next_desc);
 514                 }
 515
 516                 res_end_idx = res_cur_idx;
 517
 518                 entry_success = copy_from_mbuf_to_vring(dev, res_base_idx,
 519                         res_end_idx, pkts[pkt_idx]);
 520
 521                 rte_compiler_barrier();
 522
 523                 /*
 524                  * Wait until it's our turn to add our buffer
 525                  * to the used ring.
 526                  */
 527                 while (unlikely(vq->last_used_idx != res_base_idx))
 528                         rte_pause();
 529
 530                 *(volatile uint16_t *)&vq->used->idx += entry_success;
 531                 vq->last_used_idx = res_end_idx;
 532
 533                 /* Kick the guest if necessary. */
 534                 if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 535                         eventfd_write((int)vq->kickfd, 1);
 536         }
 537
 538         return count;
 539 }
 540
 541 /* This function works for TX packets with mergeable feature enabled. */
 542 static inline void __attribute__((always_inline))
 543 virtio_dev_merge_tx(struct virtio_net *dev, struct rte_mempool *mbuf_pool)
 544 {
 545         struct rte_mbuf *m, *prev;
 546         struct vhost_virtqueue *vq;
 547         struct vring_desc *desc;
 548         uint64_t vb_addr = 0;
 549         uint32_t head[MAX_PKT_BURST];
 550         uint32_t used_idx;
 551         uint32_t i;
 552         uint16_t free_entries, entry_success = 0;
 553         uint16_t avail_idx;
 554         uint32_t buf_size = MBUF_SIZE - (sizeof(struct rte_mbuf)
 555                         + RTE_PKTMBUF_HEADROOM);
 556
 557         vq = dev->virtqueue[VIRTIO_TXQ];
 558         avail_idx =  *((volatile uint16_t *)&vq->avail->idx);
 559
 560         /* If there are no available buffers then return. */
 561         if (vq->last_used_idx == avail_idx)
 562                 return;
 563
 564         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_tx()\n",
 565                 dev->device_fh);
 566
 567         /* Prefetch available ring to retrieve head indexes. */
 568         rte_prefetch0(&vq->avail->ring[vq->last_used_idx & (vq->size - 1)]);
 569
 570         /*get the number of free entries in the ring*/
 571         free_entries = (avail_idx - vq->last_used_idx);
 572
 573         /* Limit to MAX_PKT_BURST. */
 574         free_entries = RTE_MIN(free_entries, MAX_PKT_BURST);
 575
 576         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n",
 577                 dev->device_fh, free_entries);
 578         /* Retrieve all of the head indexes first to avoid caching issues. */
 579         for (i = 0; i < free_entries; i++)
 580                 head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)];
 581
 582         /* Prefetch descriptor index. */
 583         rte_prefetch0(&vq->desc[head[entry_success]]);
 584         rte_prefetch0(&vq->used->ring[vq->last_used_idx & (vq->size - 1)]);
 585
 586         while (entry_success < free_entries) {
 587                 uint32_t vb_avail, vb_offset;
 588                 uint32_t seg_avail, seg_offset;
 589                 uint32_t cpy_len;
 590                 uint32_t seg_num = 0;
 591                 struct rte_mbuf *cur;
 592                 uint8_t alloc_err = 0;
 593
 594                 desc = &vq->desc[head[entry_success]];
 595
 596                 /* Discard first buffer as it is the virtio header */
 597                 desc = &vq->desc[desc->next];
 598
 599                 /* Buffer address translation. */
 600                 vb_addr = gpa_to_vva(dev, desc->addr);
 601                 /* Prefetch buffer address. */
 602                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 603
 604                 used_idx = vq->last_used_idx & (vq->size - 1);
 605
 606                 if (entry_success < (free_entries - 1)) {
 607                         /* Prefetch descriptor index. */
 608                         rte_prefetch0(&vq->desc[head[entry_success+1]]);
 609                         rte_prefetch0(&vq->used->ring[(used_idx + 1) & (vq->size - 1)]);
 610                 }
 611
 612                 /* Update used index buffer information. */
 613                 vq->used->ring[used_idx].id = head[entry_success];
 614                 vq->used->ring[used_idx].len = 0;
 615
 616                 vb_offset = 0;
 617                 vb_avail = desc->len;
 618                 seg_offset = 0;
 619                 seg_avail = buf_size;
 620                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 621
 622                 PRINT_PACKET(dev, (uintptr_t)vb_addr, desc->len, 0);
 623
 624                 /* Allocate an mbuf and populate the structure. */
 625                 m = rte_pktmbuf_alloc(mbuf_pool);
 626                 if (unlikely(m == NULL)) {
 627                         RTE_LOG(ERR, VHOST_DATA,
 628                                 "Failed to allocate memory for mbuf.\n");
 629                         return;
 630                 }
 631
 632                 seg_num++;
 633                 cur = m;
 634                 prev = m;
 635                 while (cpy_len != 0) {
 636                         rte_memcpy((void *)(rte_pktmbuf_mtod(cur, char *) + seg_offset),
 637                                 (void *)((uintptr_t)(vb_addr + vb_offset)),
 638                                 cpy_len);
 639
 640                         seg_offset += cpy_len;
 641                         vb_offset += cpy_len;
 642                         vb_avail -= cpy_len;
 643                         seg_avail -= cpy_len;
 644
 645                         if (vb_avail != 0) {
 646                                 /*
 647                                  * The segment reachs to its end,
 648                                  * while the virtio buffer in TX vring has
 649                                  * more data to be copied.
 650                                  */
 651                                 cur->data_len = seg_offset;
 652                                 m->pkt_len += seg_offset;
 653                                 /* Allocate mbuf and populate the structure. */
 654                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 655                                 if (unlikely(cur == NULL)) {
 656                                         RTE_LOG(ERR, VHOST_DATA, "Failed to "
 657                                                 "allocate memory for mbuf.\n");
 658                                         rte_pktmbuf_free(m);
 659                                         alloc_err = 1;
 660                                         break;
 661                                 }
 662
 663                                 seg_num++;
 664                                 prev->next = cur;
 665                                 prev = cur;
 666                                 seg_offset = 0;
 667                                 seg_avail = buf_size;
 668                         } else {
 669                                 if (desc->flags & VRING_DESC_F_NEXT) {
 670                                         /*
 671                                          * There are more virtio buffers in
 672                                          * same vring entry need to be copied.
 673                                          */
 674                                         if (seg_avail == 0) {
 675                                                 /*
 676                                                  * The current segment hasn't
 677                                                  * room to accomodate more
 678                                                  * data.
 679                                                  */
 680                                                 cur->data_len = seg_offset;
 681                                                 m->pkt_len += seg_offset;
 682                                                 /*
 683                                                  * Allocate an mbuf and
 684                                                  * populate the structure.
 685                                                  */
 686                                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 687                                                 if (unlikely(cur == NULL)) {
 688                                                         RTE_LOG(ERR,
 689                                                                 VHOST_DATA,
 690                                                                 "Failed to "
 691                                                                 "allocate memory "
 692                                                                 "for mbuf\n");
 693                                                         rte_pktmbuf_free(m);
 694                                                         alloc_err = 1;
 695                                                         break;
 696                                                 }
 697                                                 seg_num++;
 698                                                 prev->next = cur;
 699                                                 prev = cur;
 700                                                 seg_offset = 0;
 701                                                 seg_avail = buf_size;
 702                                         }
 703
 704                                         desc = &vq->desc[desc->next];
 705
 706                                         /* Buffer address translation. */
 707                                         vb_addr = gpa_to_vva(dev, desc->addr);
 708                                         /* Prefetch buffer address. */
 709                                         rte_prefetch0((void *)(uintptr_t)vb_addr);
 710                                         vb_offset = 0;
 711                                         vb_avail = desc->len;
 712
 713                                         PRINT_PACKET(dev, (uintptr_t)vb_addr,
 714                                                 desc->len, 0);
 715                                 } else {
 716                                         /* The whole packet completes. */
 717                                         cur->data_len = seg_offset;
 718                                         m->pkt_len += seg_offset;
 719                                         vb_avail = 0;
 720                                 }
 721                         }
 722
 723                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 724                 }
 725
 726                 if (unlikely(alloc_err == 1))
 727                         break;
 728
 729                 m->nb_segs = seg_num;
 730
 731                 vq->last_used_idx++;
 732                 entry_success++;
 733                 rte_pktmbuf_free(m);
 734         }
 735
 736         rte_compiler_barrier();
 737         vq->used->idx += entry_success;
 738         /* Kick guest if required. */
 739         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 740                 eventfd_write((int)vq->kickfd, 1);
 741
 742 }