lib/librte_vhost/vhost_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <arpa/inet.h>
  35 #include <getopt.h>
  36 #include <linux/if_ether.h>
  37 #include <linux/if_vlan.h>
  38 #include <linux/virtio_net.h>
  39 #include <linux/virtio_ring.h>
  40 #include <signal.h>
  41 #include <stdint.h>
  42 #include <sys/eventfd.h>
  43 #include <sys/param.h>
  44 #include <unistd.h>
  45
  46 #include <rte_atomic.h>
  47 #include <rte_cycles.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_log.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_malloc.h>
  52
  53 #include "main.h"
  54 #include "virtio-net.h"
  55 #include "vhost-net-cdev.h"
  56
  57 #define MAX_PKT_BURST 32                /* Max burst size for RX/TX */
  58
  59 /*
  60  * Function to convert guest physical addresses to vhost virtual addresses. This
  61  * is used to convert virtio buffer addresses.
  62  */
  63 static inline uint64_t __attribute__((always_inline))
  64 gpa_to_vva(struct virtio_net *dev, uint64_t guest_pa)
  65 {
  66         struct virtio_memory_regions *region;
  67         uint32_t regionidx;
  68         uint64_t vhost_va = 0;
  69
  70         for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
  71                 region = &dev->mem->regions[regionidx];
  72                 if ((guest_pa >= region->guest_phys_address) &&
  73                         (guest_pa <= region->guest_phys_address_end)) {
  74                         vhost_va = region->address_offset + guest_pa;
  75                         break;
  76                 }
  77         }
  78         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") GPA %p| VVA %p\n",
  79                 dev->device_fh, (void*)(uintptr_t)guest_pa, (void*)(uintptr_t)vhost_va);
  80
  81         return vhost_va;
  82 }
  83
  84 /*
  85  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
  86  * be received from the physical port or from another virtio device. A packet
  87  * count is returned to indicate the number of packets that were succesfully
  88  * added to the RX queue. This function works when mergeable is disabled.
  89  */
  90 static inline uint32_t __attribute__((always_inline))
  91 virtio_dev_rx(struct virtio_net *dev, struct rte_mbuf **pkts, uint32_t count)
  92 {
  93         struct vhost_virtqueue *vq;
  94         struct vring_desc *desc;
  95         struct rte_mbuf *buff;
  96         /* The virtio_hdr is initialised to 0. */
  97         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0,0,0,0,0,0},0};
  98         uint64_t buff_addr = 0;
  99         uint64_t buff_hdr_addr = 0;
 100         uint32_t head[MAX_PKT_BURST], packet_len = 0;
 101         uint32_t head_idx, packet_success = 0;
 102         uint32_t retry = 0;
 103         uint16_t avail_idx, res_cur_idx;
 104         uint16_t res_base_idx, res_end_idx;
 105         uint16_t free_entries;
 106         uint8_t success = 0;
 107
 108         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh);
 109         vq = dev->virtqueue[VIRTIO_RXQ];
 110         count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
 111
 112         /* As many data cores may want access to available buffers, they need to be reserved. */
 113         do {
 114                 res_base_idx = vq->last_used_idx_res;
 115                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 116
 117                 free_entries = (avail_idx - res_base_idx);
 118                 /* If retry is enabled and the queue is full then we wait and retry to avoid packet loss. */
 119                 if (enable_retry && unlikely(count > free_entries)) {
 120                         for (retry = 0; retry < burst_rx_retry_num; retry++) {
 121                                 rte_delay_us(burst_rx_delay_time);
 122                                 avail_idx =
 123                                         *((volatile uint16_t *)&vq->avail->idx);
 124                                 free_entries = (avail_idx - res_base_idx);
 125                                 if (count <= free_entries)
 126                                         break;
 127                         }
 128                 }
 129
 130                 /*check that we have enough buffers*/
 131                 if (unlikely(count > free_entries))
 132                         count = free_entries;
 133
 134                 if (count == 0)
 135                         return 0;
 136
 137                 res_end_idx = res_base_idx + count;
 138                 /* vq->last_used_idx_res is atomically updated. */
 139                 success = rte_atomic16_cmpset(&vq->last_used_idx_res, res_base_idx,
 140                                                                         res_end_idx);
 141         } while (unlikely(success == 0));
 142         res_cur_idx = res_base_idx;
 143         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n", dev->device_fh, res_cur_idx, res_end_idx);
 144
 145         /* Prefetch available ring to retrieve indexes. */
 146         rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]);
 147
 148         /* Retrieve all of the head indexes first to avoid caching issues. */
 149         for (head_idx = 0; head_idx < count; head_idx++)
 150                 head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & (vq->size - 1)];
 151
 152         /*Prefetch descriptor index. */
 153         rte_prefetch0(&vq->desc[head[packet_success]]);
 154
 155         while (res_cur_idx != res_end_idx) {
 156                 /* Get descriptor from available ring */
 157                 desc = &vq->desc[head[packet_success]];
 158
 159                 buff = pkts[packet_success];
 160
 161                 /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
 162                 buff_addr = gpa_to_vva(dev, desc->addr);
 163                 /* Prefetch buffer address. */
 164                 rte_prefetch0((void*)(uintptr_t)buff_addr);
 165
 166                 /* Copy virtio_hdr to packet and increment buffer address */
 167                 buff_hdr_addr = buff_addr;
 168                 packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen;
 169
 170                 /*
 171                  * If the descriptors are chained the header and data are
 172                  * placed in separate buffers.
 173                  */
 174                 if (desc->flags & VRING_DESC_F_NEXT) {
 175                         desc->len = vq->vhost_hlen;
 176                         desc = &vq->desc[desc->next];
 177                         /* Buffer address translation. */
 178                         buff_addr = gpa_to_vva(dev, desc->addr);
 179                         desc->len = rte_pktmbuf_data_len(buff);
 180                 } else {
 181                         buff_addr += vq->vhost_hlen;
 182                         desc->len = packet_len;
 183                 }
 184
 185                 /* Update used ring with desc information */
 186                 vq->used->ring[res_cur_idx & (vq->size - 1)].id = head[packet_success];
 187                 vq->used->ring[res_cur_idx & (vq->size - 1)].len = packet_len;
 188
 189                 /* Copy mbuf data to buffer */
 190                 rte_memcpy((void *)(uintptr_t)buff_addr,
 191                         rte_pktmbuf_mtod(buff, const void *),
 192                         rte_pktmbuf_data_len(buff));
 193                 PRINT_PACKET(dev, (uintptr_t)buff_addr,
 194                         rte_pktmbuf_data_len(buff), 0);
 195
 196                 res_cur_idx++;
 197                 packet_success++;
 198
 199                 rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
 200                         (const void *)&virtio_hdr, vq->vhost_hlen);
 201
 202                 PRINT_PACKET(dev, (uintptr_t)buff_hdr_addr, vq->vhost_hlen, 1);
 203
 204                 if (res_cur_idx < res_end_idx) {
 205                         /* Prefetch descriptor index. */
 206                         rte_prefetch0(&vq->desc[head[packet_success]]);
 207                 }
 208         }
 209
 210         rte_compiler_barrier();
 211
 212         /* Wait until it's our turn to add our buffer to the used ring. */
 213         while (unlikely(vq->last_used_idx != res_base_idx))
 214                 rte_pause();
 215
 216         *(volatile uint16_t *)&vq->used->idx += count;
 217         vq->last_used_idx = res_end_idx;
 218
 219         /* Kick the guest if necessary. */
 220         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 221                 eventfd_write((int)vq->kickfd, 1);
 222         return count;
 223 }
 224
 225 static inline uint32_t __attribute__((always_inline))
 226 copy_from_mbuf_to_vring(struct virtio_net *dev,
 227         uint16_t res_base_idx, uint16_t res_end_idx,
 228         struct rte_mbuf *pkt)
 229 {
 230         uint32_t vec_idx = 0;
 231         uint32_t entry_success = 0;
 232         struct vhost_virtqueue *vq;
 233         /* The virtio_hdr is initialised to 0. */
 234         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {
 235                 {0, 0, 0, 0, 0, 0}, 0};
 236         uint16_t cur_idx = res_base_idx;
 237         uint64_t vb_addr = 0;
 238         uint64_t vb_hdr_addr = 0;
 239         uint32_t seg_offset = 0;
 240         uint32_t vb_offset = 0;
 241         uint32_t seg_avail;
 242         uint32_t vb_avail;
 243         uint32_t cpy_len, entry_len;
 244
 245         if (pkt == NULL)
 246                 return 0;
 247
 248         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| "
 249                 "End Index %d\n",
 250                 dev->device_fh, cur_idx, res_end_idx);
 251
 252         /*
 253          * Convert from gpa to vva
 254          * (guest physical addr -> vhost virtual addr)
 255          */
 256         vq = dev->virtqueue[VIRTIO_RXQ];
 257         vb_addr =
 258                 gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 259         vb_hdr_addr = vb_addr;
 260
 261         /* Prefetch buffer address. */
 262         rte_prefetch0((void *)(uintptr_t)vb_addr);
 263
 264         virtio_hdr.num_buffers = res_end_idx - res_base_idx;
 265
 266         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") RX: Num merge buffers %d\n",
 267                 dev->device_fh, virtio_hdr.num_buffers);
 268
 269         rte_memcpy((void *)(uintptr_t)vb_hdr_addr,
 270                 (const void *)&virtio_hdr, vq->vhost_hlen);
 271
 272         PRINT_PACKET(dev, (uintptr_t)vb_hdr_addr, vq->vhost_hlen, 1);
 273
 274         seg_avail = rte_pktmbuf_data_len(pkt);
 275         vb_offset = vq->vhost_hlen;
 276         vb_avail =
 277                 vq->buf_vec[vec_idx].buf_len - vq->vhost_hlen;
 278
 279         entry_len = vq->vhost_hlen;
 280
 281         if (vb_avail == 0) {
 282                 uint32_t desc_idx =
 283                         vq->buf_vec[vec_idx].desc_idx;
 284                 vq->desc[desc_idx].len = vq->vhost_hlen;
 285
 286                 if ((vq->desc[desc_idx].flags
 287                         & VRING_DESC_F_NEXT) == 0) {
 288                         /* Update used ring with desc information */
 289                         vq->used->ring[cur_idx & (vq->size - 1)].id
 290                                 = vq->buf_vec[vec_idx].desc_idx;
 291                         vq->used->ring[cur_idx & (vq->size - 1)].len
 292                                 = entry_len;
 293
 294                         entry_len = 0;
 295                         cur_idx++;
 296                         entry_success++;
 297                 }
 298
 299                 vec_idx++;
 300                 vb_addr =
 301                         gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 302
 303                 /* Prefetch buffer address. */
 304                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 305                 vb_offset = 0;
 306                 vb_avail = vq->buf_vec[vec_idx].buf_len;
 307         }
 308
 309         cpy_len = RTE_MIN(vb_avail, seg_avail);
 310
 311         while (cpy_len > 0) {
 312                 /* Copy mbuf data to vring buffer */
 313                 rte_memcpy((void *)(uintptr_t)(vb_addr + vb_offset),
 314                         (const void *)(rte_pktmbuf_mtod(pkt, char*) + seg_offset),
 315                         cpy_len);
 316
 317                 PRINT_PACKET(dev,
 318                         (uintptr_t)(vb_addr + vb_offset),
 319                         cpy_len, 0);
 320
 321                 seg_offset += cpy_len;
 322                 vb_offset += cpy_len;
 323                 seg_avail -= cpy_len;
 324                 vb_avail -= cpy_len;
 325                 entry_len += cpy_len;
 326
 327                 if (seg_avail != 0) {
 328                         /*
 329                          * The virtio buffer in this vring
 330                          * entry reach to its end.
 331                          * But the segment doesn't complete.
 332                          */
 333                         if ((vq->desc[vq->buf_vec[vec_idx].desc_idx].flags &
 334                                 VRING_DESC_F_NEXT) == 0) {
 335                                 /* Update used ring with desc information */
 336                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 337                                         = vq->buf_vec[vec_idx].desc_idx;
 338                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 339                                         = entry_len;
 340                                 entry_len = 0;
 341                                 cur_idx++;
 342                                 entry_success++;
 343                         }
 344
 345                         vec_idx++;
 346                         vb_addr = gpa_to_vva(dev,
 347                                 vq->buf_vec[vec_idx].buf_addr);
 348                         vb_offset = 0;
 349                         vb_avail = vq->buf_vec[vec_idx].buf_len;
 350                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 351                 } else {
 352                         /*
 353                          * This current segment complete, need continue to
 354                          * check if the whole packet complete or not.
 355                          */
 356                         pkt = pkt->next;
 357                         if (pkt != NULL) {
 358                                 /*
 359                                  * There are more segments.
 360                                  */
 361                                 if (vb_avail == 0) {
 362                                         /*
 363                                          * This current buffer from vring is
 364                                          * used up, need fetch next buffer
 365                                          * from buf_vec.
 366                                          */
 367                                         uint32_t desc_idx =
 368                                                 vq->buf_vec[vec_idx].desc_idx;
 369                                         vq->desc[desc_idx].len = vb_offset;
 370
 371                                         if ((vq->desc[desc_idx].flags &
 372                                                 VRING_DESC_F_NEXT) == 0) {
 373                                                 uint16_t wrapped_idx =
 374                                                         cur_idx & (vq->size - 1);
 375                                                 /*
 376                                                  * Update used ring with the
 377                                                  * descriptor information
 378                                                  */
 379                                                 vq->used->ring[wrapped_idx].id
 380                                                         = desc_idx;
 381                                                 vq->used->ring[wrapped_idx].len
 382                                                         = entry_len;
 383                                                 entry_success++;
 384                                                 entry_len = 0;
 385                                                 cur_idx++;
 386                                         }
 387
 388                                         /* Get next buffer from buf_vec. */
 389                                         vec_idx++;
 390                                         vb_addr = gpa_to_vva(dev,
 391                                                 vq->buf_vec[vec_idx].buf_addr);
 392                                         vb_avail =
 393                                                 vq->buf_vec[vec_idx].buf_len;
 394                                         vb_offset = 0;
 395                                 }
 396
 397                                 seg_offset = 0;
 398                                 seg_avail = rte_pktmbuf_data_len(pkt);
 399                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 400                         } else {
 401                                 /*
 402                                  * This whole packet completes.
 403                                  */
 404                                 uint32_t desc_idx =
 405                                         vq->buf_vec[vec_idx].desc_idx;
 406                                 vq->desc[desc_idx].len = vb_offset;
 407
 408                                 while (vq->desc[desc_idx].flags &
 409                                         VRING_DESC_F_NEXT) {
 410                                         desc_idx = vq->desc[desc_idx].next;
 411                                          vq->desc[desc_idx].len = 0;
 412                                 }
 413
 414                                 /* Update used ring with desc information */
 415                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 416                                         = vq->buf_vec[vec_idx].desc_idx;
 417                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 418                                         = entry_len;
 419                                 entry_len = 0;
 420                                 cur_idx++;
 421                                 entry_success++;
 422                                 seg_avail = 0;
 423                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 424                         }
 425                 }
 426         }
 427
 428         return entry_success;
 429 }
 430
 431 /*
 432  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
 433  * be received from the physical port or from another virtio device. A packet
 434  * count is returned to indicate the number of packets that were succesfully
 435  * added to the RX queue. This function works for mergeable RX.
 436  */
 437 static inline uint32_t __attribute__((always_inline))
 438 virtio_dev_merge_rx(struct virtio_net *dev, struct rte_mbuf **pkts,
 439         uint32_t count)
 440 {
 441         struct vhost_virtqueue *vq;
 442         uint32_t pkt_idx = 0, entry_success = 0;
 443         uint32_t retry = 0;
 444         uint16_t avail_idx, res_cur_idx;
 445         uint16_t res_base_idx, res_end_idx;
 446         uint8_t success = 0;
 447
 448         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_rx()\n",
 449                 dev->device_fh);
 450         vq = dev->virtqueue[VIRTIO_RXQ];
 451         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
 452
 453         if (count == 0)
 454                 return 0;
 455
 456         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 457                 uint32_t secure_len = 0;
 458                 uint16_t need_cnt;
 459                 uint32_t vec_idx = 0;
 460                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vhost_hlen;
 461                 uint16_t i, id;
 462
 463                 do {
 464                         /*
 465                          * As many data cores may want access to available
 466                          * buffers, they need to be reserved.
 467                          */
 468                         res_base_idx = vq->last_used_idx_res;
 469                         res_cur_idx = res_base_idx;
 470
 471                         do {
 472                                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 473                                 if (unlikely(res_cur_idx == avail_idx)) {
 474                                         /*
 475                                          * If retry is enabled and the queue is
 476                                          * full then we wait and retry to avoid
 477                                          * packet loss.
 478                                          */
 479                                         if (enable_retry) {
 480                                                 uint8_t cont = 0;
 481                                                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
 482                                                         rte_delay_us(burst_rx_delay_time);
 483                                                         avail_idx =
 484                                                                 *((volatile uint16_t *)&vq->avail->idx);
 485                                                         if (likely(res_cur_idx != avail_idx)) {
 486                                                                 cont = 1;
 487                                                                 break;
 488                                                         }
 489                                                 }
 490                                                 if (cont == 1)
 491                                                         continue;
 492                                         }
 493
 494                                         LOG_DEBUG(VHOST_DATA,
 495                                                 "(%"PRIu64") Failed "
 496                                                 "to get enough desc from "
 497                                                 "vring\n",
 498                                                 dev->device_fh);
 499                                         return pkt_idx;
 500                                 } else {
 501                                         uint16_t wrapped_idx =
 502                                                 (res_cur_idx) & (vq->size - 1);
 503                                         uint32_t idx =
 504                                                 vq->avail->ring[wrapped_idx];
 505                                         uint8_t next_desc;
 506
 507                                         do {
 508                                                 next_desc = 0;
 509                                                 secure_len += vq->desc[idx].len;
 510                                                 if (vq->desc[idx].flags &
 511                                                         VRING_DESC_F_NEXT) {
 512                                                         idx = vq->desc[idx].next;
 513                                                         next_desc = 1;
 514                                                 }
 515                                         } while (next_desc);
 516
 517                                         res_cur_idx++;
 518                                 }
 519                         } while (pkt_len > secure_len);
 520
 521                         /* vq->last_used_idx_res is atomically updated. */
 522                         success = rte_atomic16_cmpset(&vq->last_used_idx_res,
 523                                                         res_base_idx,
 524                                                         res_cur_idx);
 525                 } while (success == 0);
 526
 527                 id = res_base_idx;
 528                 need_cnt = res_cur_idx - res_base_idx;
 529
 530                 for (i = 0; i < need_cnt; i++, id++) {
 531                         uint16_t wrapped_idx = id & (vq->size - 1);
 532                         uint32_t idx = vq->avail->ring[wrapped_idx];
 533                         uint8_t next_desc;
 534                         do {
 535                                 next_desc = 0;
 536                                 vq->buf_vec[vec_idx].buf_addr =
 537                                         vq->desc[idx].addr;
 538                                 vq->buf_vec[vec_idx].buf_len =
 539                                         vq->desc[idx].len;
 540                                 vq->buf_vec[vec_idx].desc_idx = idx;
 541                                 vec_idx++;
 542
 543                                 if (vq->desc[idx].flags & VRING_DESC_F_NEXT) {
 544                                         idx = vq->desc[idx].next;
 545                                         next_desc = 1;
 546                                 }
 547                         } while (next_desc);
 548                 }
 549
 550                 res_end_idx = res_cur_idx;
 551
 552                 entry_success = copy_from_mbuf_to_vring(dev, res_base_idx,
 553                         res_end_idx, pkts[pkt_idx]);
 554
 555                 rte_compiler_barrier();
 556
 557                 /*
 558                  * Wait until it's our turn to add our buffer
 559                  * to the used ring.
 560                  */
 561                 while (unlikely(vq->last_used_idx != res_base_idx))
 562                         rte_pause();
 563
 564                 *(volatile uint16_t *)&vq->used->idx += entry_success;
 565                 vq->last_used_idx = res_end_idx;
 566
 567                 /* Kick the guest if necessary. */
 568                 if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 569                         eventfd_write((int)vq->kickfd, 1);
 570         }
 571
 572         return count;
 573 }
 574
 575 /* This function works for TX packets with mergeable feature enabled. */
 576 static inline void __attribute__((always_inline))
 577 virtio_dev_merge_tx(struct virtio_net *dev, struct rte_mempool *mbuf_pool)
 578 {
 579         struct rte_mbuf *m, *prev;
 580         struct vhost_virtqueue *vq;
 581         struct vring_desc *desc;
 582         uint64_t vb_addr = 0;
 583         uint32_t head[MAX_PKT_BURST];
 584         uint32_t used_idx;
 585         uint32_t i;
 586         uint16_t free_entries, entry_success = 0;
 587         uint16_t avail_idx;
 588         uint32_t buf_size = MBUF_SIZE - (sizeof(struct rte_mbuf)
 589                         + RTE_PKTMBUF_HEADROOM);
 590
 591         vq = dev->virtqueue[VIRTIO_TXQ];
 592         avail_idx =  *((volatile uint16_t *)&vq->avail->idx);
 593
 594         /* If there are no available buffers then return. */
 595         if (vq->last_used_idx == avail_idx)
 596                 return;
 597
 598         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_tx()\n",
 599                 dev->device_fh);
 600
 601         /* Prefetch available ring to retrieve head indexes. */
 602         rte_prefetch0(&vq->avail->ring[vq->last_used_idx & (vq->size - 1)]);
 603
 604         /*get the number of free entries in the ring*/
 605         free_entries = (avail_idx - vq->last_used_idx);
 606
 607         /* Limit to MAX_PKT_BURST. */
 608         free_entries = RTE_MIN(free_entries, MAX_PKT_BURST);
 609
 610         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n",
 611                 dev->device_fh, free_entries);
 612         /* Retrieve all of the head indexes first to avoid caching issues. */
 613         for (i = 0; i < free_entries; i++)
 614                 head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)];
 615
 616         /* Prefetch descriptor index. */
 617         rte_prefetch0(&vq->desc[head[entry_success]]);
 618         rte_prefetch0(&vq->used->ring[vq->last_used_idx & (vq->size - 1)]);
 619
 620         while (entry_success < free_entries) {
 621                 uint32_t vb_avail, vb_offset;
 622                 uint32_t seg_avail, seg_offset;
 623                 uint32_t cpy_len;
 624                 uint32_t seg_num = 0;
 625                 struct rte_mbuf *cur;
 626                 uint8_t alloc_err = 0;
 627
 628                 desc = &vq->desc[head[entry_success]];
 629
 630                 /* Discard first buffer as it is the virtio header */
 631                 desc = &vq->desc[desc->next];
 632
 633                 /* Buffer address translation. */
 634                 vb_addr = gpa_to_vva(dev, desc->addr);
 635                 /* Prefetch buffer address. */
 636                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 637
 638                 used_idx = vq->last_used_idx & (vq->size - 1);
 639
 640                 if (entry_success < (free_entries - 1)) {
 641                         /* Prefetch descriptor index. */
 642                         rte_prefetch0(&vq->desc[head[entry_success+1]]);
 643                         rte_prefetch0(&vq->used->ring[(used_idx + 1) & (vq->size - 1)]);
 644                 }
 645
 646                 /* Update used index buffer information. */
 647                 vq->used->ring[used_idx].id = head[entry_success];
 648                 vq->used->ring[used_idx].len = 0;
 649
 650                 vb_offset = 0;
 651                 vb_avail = desc->len;
 652                 seg_offset = 0;
 653                 seg_avail = buf_size;
 654                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 655
 656                 PRINT_PACKET(dev, (uintptr_t)vb_addr, desc->len, 0);
 657
 658                 /* Allocate an mbuf and populate the structure. */
 659                 m = rte_pktmbuf_alloc(mbuf_pool);
 660                 if (unlikely(m == NULL)) {
 661                         RTE_LOG(ERR, VHOST_DATA,
 662                                 "Failed to allocate memory for mbuf.\n");
 663                         return;
 664                 }
 665
 666                 seg_num++;
 667                 cur = m;
 668                 prev = m;
 669                 while (cpy_len != 0) {
 670                         rte_memcpy((void *)(rte_pktmbuf_mtod(cur, char *) + seg_offset),
 671                                 (void *)((uintptr_t)(vb_addr + vb_offset)),
 672                                 cpy_len);
 673
 674                         seg_offset += cpy_len;
 675                         vb_offset += cpy_len;
 676                         vb_avail -= cpy_len;
 677                         seg_avail -= cpy_len;
 678
 679                         if (vb_avail != 0) {
 680                                 /*
 681                                  * The segment reachs to its end,
 682                                  * while the virtio buffer in TX vring has
 683                                  * more data to be copied.
 684                                  */
 685                                 cur->data_len = seg_offset;
 686                                 m->pkt_len += seg_offset;
 687                                 /* Allocate mbuf and populate the structure. */
 688                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 689                                 if (unlikely(cur == NULL)) {
 690                                         RTE_LOG(ERR, VHOST_DATA, "Failed to "
 691                                                 "allocate memory for mbuf.\n");
 692                                         rte_pktmbuf_free(m);
 693                                         alloc_err = 1;
 694                                         break;
 695                                 }
 696
 697                                 seg_num++;
 698                                 prev->next = cur;
 699                                 prev = cur;
 700                                 seg_offset = 0;
 701                                 seg_avail = buf_size;
 702                         } else {
 703                                 if (desc->flags & VRING_DESC_F_NEXT) {
 704                                         /*
 705                                          * There are more virtio buffers in
 706                                          * same vring entry need to be copied.
 707                                          */
 708                                         if (seg_avail == 0) {
 709                                                 /*
 710                                                  * The current segment hasn't
 711                                                  * room to accomodate more
 712                                                  * data.
 713                                                  */
 714                                                 cur->data_len = seg_offset;
 715                                                 m->pkt_len += seg_offset;
 716                                                 /*
 717                                                  * Allocate an mbuf and
 718                                                  * populate the structure.
 719                                                  */
 720                                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 721                                                 if (unlikely(cur == NULL)) {
 722                                                         RTE_LOG(ERR,
 723                                                                 VHOST_DATA,
 724                                                                 "Failed to "
 725                                                                 "allocate memory "
 726                                                                 "for mbuf\n");
 727                                                         rte_pktmbuf_free(m);
 728                                                         alloc_err = 1;
 729                                                         break;
 730                                                 }
 731                                                 seg_num++;
 732                                                 prev->next = cur;
 733                                                 prev = cur;
 734                                                 seg_offset = 0;
 735                                                 seg_avail = buf_size;
 736                                         }
 737
 738                                         desc = &vq->desc[desc->next];
 739
 740                                         /* Buffer address translation. */
 741                                         vb_addr = gpa_to_vva(dev, desc->addr);
 742                                         /* Prefetch buffer address. */
 743                                         rte_prefetch0((void *)(uintptr_t)vb_addr);
 744                                         vb_offset = 0;
 745                                         vb_avail = desc->len;
 746
 747                                         PRINT_PACKET(dev, (uintptr_t)vb_addr,
 748                                                 desc->len, 0);
 749                                 } else {
 750                                         /* The whole packet completes. */
 751                                         cur->data_len = seg_offset;
 752                                         m->pkt_len += seg_offset;
 753                                         vb_avail = 0;
 754                                 }
 755                         }
 756
 757                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 758                 }
 759
 760                 if (unlikely(alloc_err == 1))
 761                         break;
 762
 763                 m->nb_segs = seg_num;
 764
 765                 /*
 766                  * If this is the first received packet we need to learn
 767                  * the MAC and setup VMDQ
 768                  */
 769                 if (dev->ready == DEVICE_MAC_LEARNING) {
 770                         if (dev->remove || (link_vmdq(dev, m) == -1)) {
 771                                 /*
 772                                  * Discard frame if device is scheduled for
 773                                  * removal or a duplicate MAC address is found.
 774                                  */
 775                                 entry_success = free_entries;
 776                                 vq->last_used_idx += entry_success;
 777                                 rte_pktmbuf_free(m);
 778                                 break;
 779                         }
 780                 }
 781
 782                 virtio_tx_route(dev, m, mbuf_pool, (uint16_t)dev->device_fh);
 783                 vq->last_used_idx++;
 784                 entry_success++;
 785                 rte_pktmbuf_free(m);
 786         }
 787
 788         rte_compiler_barrier();
 789         vq->used->idx += entry_success;
 790         /* Kick guest if required. */
 791         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 792                 eventfd_write((int)vq->kickfd, 1);
 793
 794 }