lib/librte_vhost/vhost_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <linux/virtio_net.h>
  36
  37 #include <rte_mbuf.h>
  38 #include <rte_memcpy.h>
  39 #include <rte_virtio_net.h>
  40
  41 #include "vhost-net-cdev.h"
  42
  43 #define MAX_PKT_BURST 32                /* Max burst size for RX/TX */
  44
  45 /*
  46  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
  47  * be received from the physical port or from another virtio device. A packet
  48  * count is returned to indicate the number of packets that were succesfully
  49  * added to the RX queue. This function works when mergeable is disabled.
  50  */
  51 static inline uint32_t __attribute__((always_inline))
  52 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, uint32_t count)
  53 {
  54         struct vhost_virtqueue *vq;
  55         struct vring_desc *desc;
  56         struct rte_mbuf *buff;
  57         /* The virtio_hdr is initialised to 0. */
  58         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0,0,0,0,0,0},0};
  59         uint64_t buff_addr = 0;
  60         uint64_t buff_hdr_addr = 0;
  61         uint32_t head[MAX_PKT_BURST], packet_len = 0;
  62         uint32_t head_idx, packet_success = 0;
  63         uint16_t avail_idx, res_cur_idx;
  64         uint16_t res_base_idx, res_end_idx;
  65         uint16_t free_entries;
  66         uint8_t success = 0;
  67
  68         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh);
  69         if (unlikely(queue_id != VIRTIO_RXQ)) {
  70                 LOG_DEBUG(VHOST_DATA, "mq isn't supported in this version.\n");
  71                 return 0;
  72         }
  73
  74         vq = dev->virtqueue[VIRTIO_RXQ];
  75         count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
  76
  77         /* As many data cores may want access to available buffers, they need to be reserved. */
  78         do {
  79                 res_base_idx = vq->last_used_idx_res;
  80                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
  81
  82                 free_entries = (avail_idx - res_base_idx);
  83                 /*check that we have enough buffers*/
  84                 if (unlikely(count > free_entries))
  85                         count = free_entries;
  86
  87                 if (count == 0)
  88                         return 0;
  89
  90                 res_end_idx = res_base_idx + count;
  91                 /* vq->last_used_idx_res is atomically updated. */
  92                 success = rte_atomic16_cmpset(&vq->last_used_idx_res, res_base_idx,
  93                                                                         res_end_idx);
  94         } while (unlikely(success == 0));
  95         res_cur_idx = res_base_idx;
  96         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n", dev->device_fh, res_cur_idx, res_end_idx);
  97
  98         /* Prefetch available ring to retrieve indexes. */
  99         rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]);
 100
 101         /* Retrieve all of the head indexes first to avoid caching issues. */
 102         for (head_idx = 0; head_idx < count; head_idx++)
 103                 head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & (vq->size - 1)];
 104
 105         /*Prefetch descriptor index. */
 106         rte_prefetch0(&vq->desc[head[packet_success]]);
 107
 108         while (res_cur_idx != res_end_idx) {
 109                 /* Get descriptor from available ring */
 110                 desc = &vq->desc[head[packet_success]];
 111
 112                 buff = pkts[packet_success];
 113
 114                 /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
 115                 buff_addr = gpa_to_vva(dev, desc->addr);
 116                 /* Prefetch buffer address. */
 117                 rte_prefetch0((void*)(uintptr_t)buff_addr);
 118
 119                 /* Copy virtio_hdr to packet and increment buffer address */
 120                 buff_hdr_addr = buff_addr;
 121                 packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen;
 122
 123                 /*
 124                  * If the descriptors are chained the header and data are
 125                  * placed in separate buffers.
 126                  */
 127                 if (desc->flags & VRING_DESC_F_NEXT) {
 128                         desc->len = vq->vhost_hlen;
 129                         desc = &vq->desc[desc->next];
 130                         /* Buffer address translation. */
 131                         buff_addr = gpa_to_vva(dev, desc->addr);
 132                         desc->len = rte_pktmbuf_data_len(buff);
 133                 } else {
 134                         buff_addr += vq->vhost_hlen;
 135                         desc->len = packet_len;
 136                 }
 137
 138                 /* Update used ring with desc information */
 139                 vq->used->ring[res_cur_idx & (vq->size - 1)].id = head[packet_success];
 140                 vq->used->ring[res_cur_idx & (vq->size - 1)].len = packet_len;
 141
 142                 /* Copy mbuf data to buffer */
 143                 rte_memcpy((void *)(uintptr_t)buff_addr,
 144                         rte_pktmbuf_mtod(buff, const void *),
 145                         rte_pktmbuf_data_len(buff));
 146                 PRINT_PACKET(dev, (uintptr_t)buff_addr,
 147                         rte_pktmbuf_data_len(buff), 0);
 148
 149                 res_cur_idx++;
 150                 packet_success++;
 151
 152                 rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
 153                         (const void *)&virtio_hdr, vq->vhost_hlen);
 154
 155                 PRINT_PACKET(dev, (uintptr_t)buff_hdr_addr, vq->vhost_hlen, 1);
 156
 157                 if (res_cur_idx < res_end_idx) {
 158                         /* Prefetch descriptor index. */
 159                         rte_prefetch0(&vq->desc[head[packet_success]]);
 160                 }
 161         }
 162
 163         rte_compiler_barrier();
 164
 165         /* Wait until it's our turn to add our buffer to the used ring. */
 166         while (unlikely(vq->last_used_idx != res_base_idx))
 167                 rte_pause();
 168
 169         *(volatile uint16_t *)&vq->used->idx += count;
 170         vq->last_used_idx = res_end_idx;
 171
 172         /* Kick the guest if necessary. */
 173         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 174                 eventfd_write((int)vq->kickfd, 1);
 175         return count;
 176 }
 177
 178 static inline uint32_t __attribute__((always_inline))
 179 copy_from_mbuf_to_vring(struct virtio_net *dev,
 180         uint16_t res_base_idx, uint16_t res_end_idx,
 181         struct rte_mbuf *pkt)
 182 {
 183         uint32_t vec_idx = 0;
 184         uint32_t entry_success = 0;
 185         struct vhost_virtqueue *vq;
 186         /* The virtio_hdr is initialised to 0. */
 187         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {
 188                 {0, 0, 0, 0, 0, 0}, 0};
 189         uint16_t cur_idx = res_base_idx;
 190         uint64_t vb_addr = 0;
 191         uint64_t vb_hdr_addr = 0;
 192         uint32_t seg_offset = 0;
 193         uint32_t vb_offset = 0;
 194         uint32_t seg_avail;
 195         uint32_t vb_avail;
 196         uint32_t cpy_len, entry_len;
 197
 198         if (pkt == NULL)
 199                 return 0;
 200
 201         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| "
 202                 "End Index %d\n",
 203                 dev->device_fh, cur_idx, res_end_idx);
 204
 205         /*
 206          * Convert from gpa to vva
 207          * (guest physical addr -> vhost virtual addr)
 208          */
 209         vq = dev->virtqueue[VIRTIO_RXQ];
 210         vb_addr =
 211                 gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 212         vb_hdr_addr = vb_addr;
 213
 214         /* Prefetch buffer address. */
 215         rte_prefetch0((void *)(uintptr_t)vb_addr);
 216
 217         virtio_hdr.num_buffers = res_end_idx - res_base_idx;
 218
 219         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") RX: Num merge buffers %d\n",
 220                 dev->device_fh, virtio_hdr.num_buffers);
 221
 222         rte_memcpy((void *)(uintptr_t)vb_hdr_addr,
 223                 (const void *)&virtio_hdr, vq->vhost_hlen);
 224
 225         PRINT_PACKET(dev, (uintptr_t)vb_hdr_addr, vq->vhost_hlen, 1);
 226
 227         seg_avail = rte_pktmbuf_data_len(pkt);
 228         vb_offset = vq->vhost_hlen;
 229         vb_avail =
 230                 vq->buf_vec[vec_idx].buf_len - vq->vhost_hlen;
 231
 232         entry_len = vq->vhost_hlen;
 233
 234         if (vb_avail == 0) {
 235                 uint32_t desc_idx =
 236                         vq->buf_vec[vec_idx].desc_idx;
 237                 vq->desc[desc_idx].len = vq->vhost_hlen;
 238
 239                 if ((vq->desc[desc_idx].flags
 240                         & VRING_DESC_F_NEXT) == 0) {
 241                         /* Update used ring with desc information */
 242                         vq->used->ring[cur_idx & (vq->size - 1)].id
 243                                 = vq->buf_vec[vec_idx].desc_idx;
 244                         vq->used->ring[cur_idx & (vq->size - 1)].len
 245                                 = entry_len;
 246
 247                         entry_len = 0;
 248                         cur_idx++;
 249                         entry_success++;
 250                 }
 251
 252                 vec_idx++;
 253                 vb_addr =
 254                         gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 255
 256                 /* Prefetch buffer address. */
 257                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 258                 vb_offset = 0;
 259                 vb_avail = vq->buf_vec[vec_idx].buf_len;
 260         }
 261
 262         cpy_len = RTE_MIN(vb_avail, seg_avail);
 263
 264         while (cpy_len > 0) {
 265                 /* Copy mbuf data to vring buffer */
 266                 rte_memcpy((void *)(uintptr_t)(vb_addr + vb_offset),
 267                         (const void *)(rte_pktmbuf_mtod(pkt, char*) + seg_offset),
 268                         cpy_len);
 269
 270                 PRINT_PACKET(dev,
 271                         (uintptr_t)(vb_addr + vb_offset),
 272                         cpy_len, 0);
 273
 274                 seg_offset += cpy_len;
 275                 vb_offset += cpy_len;
 276                 seg_avail -= cpy_len;
 277                 vb_avail -= cpy_len;
 278                 entry_len += cpy_len;
 279
 280                 if (seg_avail != 0) {
 281                         /*
 282                          * The virtio buffer in this vring
 283                          * entry reach to its end.
 284                          * But the segment doesn't complete.
 285                          */
 286                         if ((vq->desc[vq->buf_vec[vec_idx].desc_idx].flags &
 287                                 VRING_DESC_F_NEXT) == 0) {
 288                                 /* Update used ring with desc information */
 289                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 290                                         = vq->buf_vec[vec_idx].desc_idx;
 291                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 292                                         = entry_len;
 293                                 entry_len = 0;
 294                                 cur_idx++;
 295                                 entry_success++;
 296                         }
 297
 298                         vec_idx++;
 299                         vb_addr = gpa_to_vva(dev,
 300                                 vq->buf_vec[vec_idx].buf_addr);
 301                         vb_offset = 0;
 302                         vb_avail = vq->buf_vec[vec_idx].buf_len;
 303                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 304                 } else {
 305                         /*
 306                          * This current segment complete, need continue to
 307                          * check if the whole packet complete or not.
 308                          */
 309                         pkt = pkt->next;
 310                         if (pkt != NULL) {
 311                                 /*
 312                                  * There are more segments.
 313                                  */
 314                                 if (vb_avail == 0) {
 315                                         /*
 316                                          * This current buffer from vring is
 317                                          * used up, need fetch next buffer
 318                                          * from buf_vec.
 319                                          */
 320                                         uint32_t desc_idx =
 321                                                 vq->buf_vec[vec_idx].desc_idx;
 322                                         vq->desc[desc_idx].len = vb_offset;
 323
 324                                         if ((vq->desc[desc_idx].flags &
 325                                                 VRING_DESC_F_NEXT) == 0) {
 326                                                 uint16_t wrapped_idx =
 327                                                         cur_idx & (vq->size - 1);
 328                                                 /*
 329                                                  * Update used ring with the
 330                                                  * descriptor information
 331                                                  */
 332                                                 vq->used->ring[wrapped_idx].id
 333                                                         = desc_idx;
 334                                                 vq->used->ring[wrapped_idx].len
 335                                                         = entry_len;
 336                                                 entry_success++;
 337                                                 entry_len = 0;
 338                                                 cur_idx++;
 339                                         }
 340
 341                                         /* Get next buffer from buf_vec. */
 342                                         vec_idx++;
 343                                         vb_addr = gpa_to_vva(dev,
 344                                                 vq->buf_vec[vec_idx].buf_addr);
 345                                         vb_avail =
 346                                                 vq->buf_vec[vec_idx].buf_len;
 347                                         vb_offset = 0;
 348                                 }
 349
 350                                 seg_offset = 0;
 351                                 seg_avail = rte_pktmbuf_data_len(pkt);
 352                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 353                         } else {
 354                                 /*
 355                                  * This whole packet completes.
 356                                  */
 357                                 uint32_t desc_idx =
 358                                         vq->buf_vec[vec_idx].desc_idx;
 359                                 vq->desc[desc_idx].len = vb_offset;
 360
 361                                 while (vq->desc[desc_idx].flags &
 362                                         VRING_DESC_F_NEXT) {
 363                                         desc_idx = vq->desc[desc_idx].next;
 364                                          vq->desc[desc_idx].len = 0;
 365                                 }
 366
 367                                 /* Update used ring with desc information */
 368                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 369                                         = vq->buf_vec[vec_idx].desc_idx;
 370                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 371                                         = entry_len;
 372                                 entry_len = 0;
 373                                 cur_idx++;
 374                                 entry_success++;
 375                                 seg_avail = 0;
 376                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 377                         }
 378                 }
 379         }
 380
 381         return entry_success;
 382 }
 383
 384 /*
 385  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
 386  * be received from the physical port or from another virtio device. A packet
 387  * count is returned to indicate the number of packets that were succesfully
 388  * added to the RX queue. This function works for mergeable RX.
 389  */
 390 static inline uint32_t __attribute__((always_inline))
 391 virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts,
 392         uint32_t count)
 393 {
 394         struct vhost_virtqueue *vq;
 395         uint32_t pkt_idx = 0, entry_success = 0;
 396         uint16_t avail_idx, res_cur_idx;
 397         uint16_t res_base_idx, res_end_idx;
 398         uint8_t success = 0;
 399
 400         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_rx()\n",
 401                 dev->device_fh);
 402         if (unlikely(queue_id != VIRTIO_RXQ)) {
 403                 LOG_DEBUG(VHOST_DATA, "mq isn't supported in this version.\n");
 404         }
 405
 406         vq = dev->virtqueue[VIRTIO_RXQ];
 407         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
 408
 409         if (count == 0)
 410                 return 0;
 411
 412         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 413                 uint32_t secure_len = 0;
 414                 uint16_t need_cnt;
 415                 uint32_t vec_idx = 0;
 416                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vhost_hlen;
 417                 uint16_t i, id;
 418
 419                 do {
 420                         /*
 421                          * As many data cores may want access to available
 422                          * buffers, they need to be reserved.
 423                          */
 424                         res_base_idx = vq->last_used_idx_res;
 425                         res_cur_idx = res_base_idx;
 426
 427                         do {
 428                                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 429                                 if (unlikely(res_cur_idx == avail_idx)) {
 430                                         LOG_DEBUG(VHOST_DATA,
 431                                                 "(%"PRIu64") Failed "
 432                                                 "to get enough desc from "
 433                                                 "vring\n",
 434                                                 dev->device_fh);
 435                                         return pkt_idx;
 436                                 } else {
 437                                         uint16_t wrapped_idx =
 438                                                 (res_cur_idx) & (vq->size - 1);
 439                                         uint32_t idx =
 440                                                 vq->avail->ring[wrapped_idx];
 441                                         uint8_t next_desc;
 442
 443                                         do {
 444                                                 next_desc = 0;
 445                                                 secure_len += vq->desc[idx].len;
 446                                                 if (vq->desc[idx].flags &
 447                                                         VRING_DESC_F_NEXT) {
 448                                                         idx = vq->desc[idx].next;
 449                                                         next_desc = 1;
 450                                                 }
 451                                         } while (next_desc);
 452
 453                                         res_cur_idx++;
 454                                 }
 455                         } while (pkt_len > secure_len);
 456
 457                         /* vq->last_used_idx_res is atomically updated. */
 458                         success = rte_atomic16_cmpset(&vq->last_used_idx_res,
 459                                                         res_base_idx,
 460                                                         res_cur_idx);
 461                 } while (success == 0);
 462
 463                 id = res_base_idx;
 464                 need_cnt = res_cur_idx - res_base_idx;
 465
 466                 for (i = 0; i < need_cnt; i++, id++) {
 467                         uint16_t wrapped_idx = id & (vq->size - 1);
 468                         uint32_t idx = vq->avail->ring[wrapped_idx];
 469                         uint8_t next_desc;
 470                         do {
 471                                 next_desc = 0;
 472                                 vq->buf_vec[vec_idx].buf_addr =
 473                                         vq->desc[idx].addr;
 474                                 vq->buf_vec[vec_idx].buf_len =
 475                                         vq->desc[idx].len;
 476                                 vq->buf_vec[vec_idx].desc_idx = idx;
 477                                 vec_idx++;
 478
 479                                 if (vq->desc[idx].flags & VRING_DESC_F_NEXT) {
 480                                         idx = vq->desc[idx].next;
 481                                         next_desc = 1;
 482                                 }
 483                         } while (next_desc);
 484                 }
 485
 486                 res_end_idx = res_cur_idx;
 487
 488                 entry_success = copy_from_mbuf_to_vring(dev, res_base_idx,
 489                         res_end_idx, pkts[pkt_idx]);
 490
 491                 rte_compiler_barrier();
 492
 493                 /*
 494                  * Wait until it's our turn to add our buffer
 495                  * to the used ring.
 496                  */
 497                 while (unlikely(vq->last_used_idx != res_base_idx))
 498                         rte_pause();
 499
 500                 *(volatile uint16_t *)&vq->used->idx += entry_success;
 501                 vq->last_used_idx = res_end_idx;
 502
 503                 /* Kick the guest if necessary. */
 504                 if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 505                         eventfd_write((int)vq->kickfd, 1);
 506         }
 507
 508         return count;
 509 }
 510
 511 uint16_t
 512 rte_vhost_enqueue_burst(struct virtio_net *dev, uint16_t queue_id, struct rte_mbuf **pkts, uint16_t count)
 513 {
 514         if (unlikely(dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)))
 515                 return virtio_dev_merge_rx(dev, queue_id, pkts, count);
 516         else
 517                 return virtio_dev_rx(dev, queue_id, pkts, count);
 518 }
 519
 520 uint16_t
 521 rte_vhost_dequeue_burst(struct virtio_net *dev, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 522 {
 523         struct rte_mbuf *m, *prev;
 524         struct vhost_virtqueue *vq;
 525         struct vring_desc *desc;
 526         uint64_t vb_addr = 0;
 527         uint32_t head[MAX_PKT_BURST];
 528         uint32_t used_idx;
 529         uint32_t i;
 530         uint16_t free_entries, entry_success = 0;
 531         uint16_t avail_idx;
 532
 533         if (unlikely(queue_id != VIRTIO_TXQ)) {
 534                 LOG_DEBUG(VHOST_DATA, "mq isn't supported in this version.\n");
 535                 return 0;
 536         }
 537
 538         vq = dev->virtqueue[VIRTIO_TXQ];
 539         avail_idx =  *((volatile uint16_t *)&vq->avail->idx);
 540
 541         /* If there are no available buffers then return. */
 542         if (vq->last_used_idx == avail_idx)
 543                 return 0;
 544
 545         LOG_DEBUG(VHOST_DATA, "%s (%"PRIu64")\n", __func__,
 546                 dev->device_fh);
 547
 548         /* Prefetch available ring to retrieve head indexes. */
 549         rte_prefetch0(&vq->avail->ring[vq->last_used_idx & (vq->size - 1)]);
 550
 551         /*get the number of free entries in the ring*/
 552         free_entries = (avail_idx - vq->last_used_idx);
 553
 554         free_entries = RTE_MIN(free_entries, count);
 555         /* Limit to MAX_PKT_BURST. */
 556         free_entries = RTE_MIN(free_entries, MAX_PKT_BURST);
 557
 558         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n",
 559                 dev->device_fh, free_entries);
 560         /* Retrieve all of the head indexes first to avoid caching issues. */
 561         for (i = 0; i < free_entries; i++)
 562                 head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)];
 563
 564         /* Prefetch descriptor index. */
 565         rte_prefetch0(&vq->desc[head[entry_success]]);
 566         rte_prefetch0(&vq->used->ring[vq->last_used_idx & (vq->size - 1)]);
 567
 568         while (entry_success < free_entries) {
 569                 uint32_t vb_avail, vb_offset;
 570                 uint32_t seg_avail, seg_offset;
 571                 uint32_t cpy_len;
 572                 uint32_t seg_num = 0;
 573                 struct rte_mbuf *cur;
 574                 uint8_t alloc_err = 0;
 575
 576                 desc = &vq->desc[head[entry_success]];
 577
 578                 /* Discard first buffer as it is the virtio header */
 579                 desc = &vq->desc[desc->next];
 580
 581                 /* Buffer address translation. */
 582                 vb_addr = gpa_to_vva(dev, desc->addr);
 583                 /* Prefetch buffer address. */
 584                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 585
 586                 used_idx = vq->last_used_idx & (vq->size - 1);
 587
 588                 if (entry_success < (free_entries - 1)) {
 589                         /* Prefetch descriptor index. */
 590                         rte_prefetch0(&vq->desc[head[entry_success+1]]);
 591                         rte_prefetch0(&vq->used->ring[(used_idx + 1) & (vq->size - 1)]);
 592                 }
 593
 594                 /* Update used index buffer information. */
 595                 vq->used->ring[used_idx].id = head[entry_success];
 596                 vq->used->ring[used_idx].len = 0;
 597
 598                 vb_offset = 0;
 599                 vb_avail = desc->len;
 600                 /* Allocate an mbuf and populate the structure. */
 601                 m = rte_pktmbuf_alloc(mbuf_pool);
 602                 if (unlikely(m == NULL)) {
 603                         RTE_LOG(ERR, VHOST_DATA,
 604                                 "Failed to allocate memory for mbuf.\n");
 605                         return entry_success;
 606                 }
 607                 seg_offset = 0;
 608                 seg_avail = m->buf_len - RTE_PKTMBUF_HEADROOM;
 609                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 610
 611                 PRINT_PACKET(dev, (uintptr_t)vb_addr, desc->len, 0);
 612
 613                 seg_num++;
 614                 cur = m;
 615                 prev = m;
 616                 while (cpy_len != 0) {
 617                         rte_memcpy((void *)(rte_pktmbuf_mtod(cur, char *) + seg_offset),
 618                                 (void *)((uintptr_t)(vb_addr + vb_offset)),
 619                                 cpy_len);
 620
 621                         seg_offset += cpy_len;
 622                         vb_offset += cpy_len;
 623                         vb_avail -= cpy_len;
 624                         seg_avail -= cpy_len;
 625
 626                         if (vb_avail != 0) {
 627                                 /*
 628                                  * The segment reachs to its end,
 629                                  * while the virtio buffer in TX vring has
 630                                  * more data to be copied.
 631                                  */
 632                                 cur->data_len = seg_offset;
 633                                 m->pkt_len += seg_offset;
 634                                 /* Allocate mbuf and populate the structure. */
 635                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 636                                 if (unlikely(cur == NULL)) {
 637                                         RTE_LOG(ERR, VHOST_DATA, "Failed to "
 638                                                 "allocate memory for mbuf.\n");
 639                                         rte_pktmbuf_free(m);
 640                                         alloc_err = 1;
 641                                         break;
 642                                 }
 643
 644                                 seg_num++;
 645                                 prev->next = cur;
 646                                 prev = cur;
 647                                 seg_offset = 0;
 648                                 seg_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM;
 649                         } else {
 650                                 if (desc->flags & VRING_DESC_F_NEXT) {
 651                                         /*
 652                                          * There are more virtio buffers in
 653                                          * same vring entry need to be copied.
 654                                          */
 655                                         if (seg_avail == 0) {
 656                                                 /*
 657                                                  * The current segment hasn't
 658                                                  * room to accomodate more
 659                                                  * data.
 660                                                  */
 661                                                 cur->data_len = seg_offset;
 662                                                 m->pkt_len += seg_offset;
 663                                                 /*
 664                                                  * Allocate an mbuf and
 665                                                  * populate the structure.
 666                                                  */
 667                                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 668                                                 if (unlikely(cur == NULL)) {
 669                                                         RTE_LOG(ERR,
 670                                                                 VHOST_DATA,
 671                                                                 "Failed to "
 672                                                                 "allocate memory "
 673                                                                 "for mbuf\n");
 674                                                         rte_pktmbuf_free(m);
 675                                                         alloc_err = 1;
 676                                                         break;
 677                                                 }
 678                                                 seg_num++;
 679                                                 prev->next = cur;
 680                                                 prev = cur;
 681                                                 seg_offset = 0;
 682                                                 seg_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM;
 683                                         }
 684
 685                                         desc = &vq->desc[desc->next];
 686
 687                                         /* Buffer address translation. */
 688                                         vb_addr = gpa_to_vva(dev, desc->addr);
 689                                         /* Prefetch buffer address. */
 690                                         rte_prefetch0((void *)(uintptr_t)vb_addr);
 691                                         vb_offset = 0;
 692                                         vb_avail = desc->len;
 693
 694                                         PRINT_PACKET(dev, (uintptr_t)vb_addr,
 695                                                 desc->len, 0);
 696                                 } else {
 697                                         /* The whole packet completes. */
 698                                         cur->data_len = seg_offset;
 699                                         m->pkt_len += seg_offset;
 700                                         vb_avail = 0;
 701                                 }
 702                         }
 703
 704                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 705                 }
 706
 707                 if (unlikely(alloc_err == 1))
 708                         break;
 709
 710                 m->nb_segs = seg_num;
 711
 712                 pkts[entry_success] = m;
 713                 vq->last_used_idx++;
 714                 entry_success++;
 715         }
 716
 717         rte_compiler_barrier();
 718         vq->used->idx += entry_success;
 719         /* Kick guest if required. */
 720         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 721                 eventfd_write((int)vq->kickfd, 1);
 722         return entry_success;
 723
 724 }