lib/librte_vhost/vhost_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdbool.h>
  36 #include <linux/virtio_net.h>
  37
  38 #include <rte_mbuf.h>
  39 #include <rte_memcpy.h>
  40 #include <rte_ether.h>
  41 #include <rte_ip.h>
  42 #include <rte_virtio_net.h>
  43 #include <rte_tcp.h>
  44 #include <rte_udp.h>
  45 #include <rte_sctp.h>
  46
  47 #include "vhost-net.h"
  48
  49 #define MAX_PKT_BURST 32
  50
  51 static bool
  52 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t qp_nb)
  53 {
  54         return (is_tx ^ (idx & 1)) == 0 && idx < qp_nb * VIRTIO_QNUM;
  55 }
  56
  57 /**
  58  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
  59  * be received from the physical port or from another virtio device. A packet
  60  * count is returned to indicate the number of packets that are succesfully
  61  * added to the RX queue. This function works when the mbuf is scattered, but
  62  * it doesn't support the mergeable feature.
  63  */
  64 static inline uint32_t __attribute__((always_inline))
  65 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
  66         struct rte_mbuf **pkts, uint32_t count)
  67 {
  68         struct vhost_virtqueue *vq;
  69         struct vring_desc *desc;
  70         struct rte_mbuf *buff;
  71         /* The virtio_hdr is initialised to 0. */
  72         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0};
  73         uint64_t buff_addr = 0;
  74         uint64_t buff_hdr_addr = 0;
  75         uint32_t head[MAX_PKT_BURST];
  76         uint32_t head_idx, packet_success = 0;
  77         uint16_t avail_idx, res_cur_idx;
  78         uint16_t res_base_idx, res_end_idx;
  79         uint16_t free_entries;
  80         uint8_t success = 0;
  81
  82         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh);
  83         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) {
  84                 RTE_LOG(ERR, VHOST_DATA,
  85                         "%s (%"PRIu64"): virtqueue idx:%d invalid.\n",
  86                         __func__, dev->device_fh, queue_id);
  87                 return 0;
  88         }
  89
  90         vq = dev->virtqueue[queue_id];
  91         if (unlikely(vq->enabled == 0))
  92                 return 0;
  93
  94         count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
  95
  96         /*
  97          * As many data cores may want access to available buffers,
  98          * they need to be reserved.
  99          */
 100         do {
 101                 res_base_idx = vq->last_used_idx_res;
 102                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 103
 104                 free_entries = (avail_idx - res_base_idx);
 105                 /*check that we have enough buffers*/
 106                 if (unlikely(count > free_entries))
 107                         count = free_entries;
 108
 109                 if (count == 0)
 110                         return 0;
 111
 112                 res_end_idx = res_base_idx + count;
 113                 /* vq->last_used_idx_res is atomically updated. */
 114                 /* TODO: Allow to disable cmpset if no concurrency in application. */
 115                 success = rte_atomic16_cmpset(&vq->last_used_idx_res,
 116                                 res_base_idx, res_end_idx);
 117         } while (unlikely(success == 0));
 118         res_cur_idx = res_base_idx;
 119         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n",
 120                         dev->device_fh, res_cur_idx, res_end_idx);
 121
 122         /* Prefetch available ring to retrieve indexes. */
 123         rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]);
 124
 125         /* Retrieve all of the head indexes first to avoid caching issues. */
 126         for (head_idx = 0; head_idx < count; head_idx++)
 127                 head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) &
 128                                         (vq->size - 1)];
 129
 130         /*Prefetch descriptor index. */
 131         rte_prefetch0(&vq->desc[head[packet_success]]);
 132
 133         while (res_cur_idx != res_end_idx) {
 134                 uint32_t offset = 0, vb_offset = 0;
 135                 uint32_t pkt_len, len_to_cpy, data_len, total_copied = 0;
 136                 uint8_t hdr = 0, uncompleted_pkt = 0;
 137
 138                 /* Get descriptor from available ring */
 139                 desc = &vq->desc[head[packet_success]];
 140
 141                 buff = pkts[packet_success];
 142
 143                 /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
 144                 buff_addr = gpa_to_vva(dev, desc->addr);
 145                 /* Prefetch buffer address. */
 146                 rte_prefetch0((void *)(uintptr_t)buff_addr);
 147
 148                 /* Copy virtio_hdr to packet and increment buffer address */
 149                 buff_hdr_addr = buff_addr;
 150
 151                 /*
 152                  * If the descriptors are chained the header and data are
 153                  * placed in separate buffers.
 154                  */
 155                 if ((desc->flags & VRING_DESC_F_NEXT) &&
 156                         (desc->len == vq->vhost_hlen)) {
 157                         desc = &vq->desc[desc->next];
 158                         /* Buffer address translation. */
 159                         buff_addr = gpa_to_vva(dev, desc->addr);
 160                 } else {
 161                         vb_offset += vq->vhost_hlen;
 162                         hdr = 1;
 163                 }
 164
 165                 pkt_len = rte_pktmbuf_pkt_len(buff);
 166                 data_len = rte_pktmbuf_data_len(buff);
 167                 len_to_cpy = RTE_MIN(data_len,
 168                         hdr ? desc->len - vq->vhost_hlen : desc->len);
 169                 while (total_copied < pkt_len) {
 170                         /* Copy mbuf data to buffer */
 171                         rte_memcpy((void *)(uintptr_t)(buff_addr + vb_offset),
 172                                 rte_pktmbuf_mtod_offset(buff, const void *, offset),
 173                                 len_to_cpy);
 174                         PRINT_PACKET(dev, (uintptr_t)(buff_addr + vb_offset),
 175                                 len_to_cpy, 0);
 176
 177                         offset += len_to_cpy;
 178                         vb_offset += len_to_cpy;
 179                         total_copied += len_to_cpy;
 180
 181                         /* The whole packet completes */
 182                         if (total_copied == pkt_len)
 183                                 break;
 184
 185                         /* The current segment completes */
 186                         if (offset == data_len) {
 187                                 buff = buff->next;
 188                                 offset = 0;
 189                                 data_len = rte_pktmbuf_data_len(buff);
 190                         }
 191
 192                         /* The current vring descriptor done */
 193                         if (vb_offset == desc->len) {
 194                                 if (desc->flags & VRING_DESC_F_NEXT) {
 195                                         desc = &vq->desc[desc->next];
 196                                         buff_addr = gpa_to_vva(dev, desc->addr);
 197                                         vb_offset = 0;
 198                                 } else {
 199                                         /* Room in vring buffer is not enough */
 200                                         uncompleted_pkt = 1;
 201                                         break;
 202                                 }
 203                         }
 204                         len_to_cpy = RTE_MIN(data_len - offset, desc->len - vb_offset);
 205                 }
 206
 207                 /* Update used ring with desc information */
 208                 vq->used->ring[res_cur_idx & (vq->size - 1)].id =
 209                                                         head[packet_success];
 210
 211                 /* Drop the packet if it is uncompleted */
 212                 if (unlikely(uncompleted_pkt == 1))
 213                         vq->used->ring[res_cur_idx & (vq->size - 1)].len =
 214                                                         vq->vhost_hlen;
 215                 else
 216                         vq->used->ring[res_cur_idx & (vq->size - 1)].len =
 217                                                         pkt_len + vq->vhost_hlen;
 218
 219                 res_cur_idx++;
 220                 packet_success++;
 221
 222                 if (unlikely(uncompleted_pkt == 1))
 223                         continue;
 224
 225                 rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
 226                         (const void *)&virtio_hdr, vq->vhost_hlen);
 227
 228                 PRINT_PACKET(dev, (uintptr_t)buff_hdr_addr, vq->vhost_hlen, 1);
 229
 230                 if (res_cur_idx < res_end_idx) {
 231                         /* Prefetch descriptor index. */
 232                         rte_prefetch0(&vq->desc[head[packet_success]]);
 233                 }
 234         }
 235
 236         rte_compiler_barrier();
 237
 238         /* Wait until it's our turn to add our buffer to the used ring. */
 239         while (unlikely(vq->last_used_idx != res_base_idx))
 240                 rte_pause();
 241
 242         *(volatile uint16_t *)&vq->used->idx += count;
 243         vq->last_used_idx = res_end_idx;
 244
 245         /* flush used->idx update before we read avail->flags. */
 246         rte_mb();
 247
 248         /* Kick the guest if necessary. */
 249         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 250                 eventfd_write(vq->callfd, (eventfd_t)1);
 251         return count;
 252 }
 253
 254 static inline uint32_t __attribute__((always_inline))
 255 copy_from_mbuf_to_vring(struct virtio_net *dev, uint32_t queue_id,
 256                         uint16_t res_base_idx, uint16_t res_end_idx,
 257                         struct rte_mbuf *pkt)
 258 {
 259         uint32_t vec_idx = 0;
 260         uint32_t entry_success = 0;
 261         struct vhost_virtqueue *vq;
 262         /* The virtio_hdr is initialised to 0. */
 263         struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {
 264                 {0, 0, 0, 0, 0, 0}, 0};
 265         uint16_t cur_idx = res_base_idx;
 266         uint64_t vb_addr = 0;
 267         uint64_t vb_hdr_addr = 0;
 268         uint32_t seg_offset = 0;
 269         uint32_t vb_offset = 0;
 270         uint32_t seg_avail;
 271         uint32_t vb_avail;
 272         uint32_t cpy_len, entry_len;
 273
 274         if (pkt == NULL)
 275                 return 0;
 276
 277         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| "
 278                 "End Index %d\n",
 279                 dev->device_fh, cur_idx, res_end_idx);
 280
 281         /*
 282          * Convert from gpa to vva
 283          * (guest physical addr -> vhost virtual addr)
 284          */
 285         vq = dev->virtqueue[queue_id];
 286
 287         vb_addr = gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 288         vb_hdr_addr = vb_addr;
 289
 290         /* Prefetch buffer address. */
 291         rte_prefetch0((void *)(uintptr_t)vb_addr);
 292
 293         virtio_hdr.num_buffers = res_end_idx - res_base_idx;
 294
 295         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") RX: Num merge buffers %d\n",
 296                 dev->device_fh, virtio_hdr.num_buffers);
 297
 298         rte_memcpy((void *)(uintptr_t)vb_hdr_addr,
 299                 (const void *)&virtio_hdr, vq->vhost_hlen);
 300
 301         PRINT_PACKET(dev, (uintptr_t)vb_hdr_addr, vq->vhost_hlen, 1);
 302
 303         seg_avail = rte_pktmbuf_data_len(pkt);
 304         vb_offset = vq->vhost_hlen;
 305         vb_avail = vq->buf_vec[vec_idx].buf_len - vq->vhost_hlen;
 306
 307         entry_len = vq->vhost_hlen;
 308
 309         if (vb_avail == 0) {
 310                 uint32_t desc_idx =
 311                         vq->buf_vec[vec_idx].desc_idx;
 312
 313                 if ((vq->desc[desc_idx].flags
 314                         & VRING_DESC_F_NEXT) == 0) {
 315                         /* Update used ring with desc information */
 316                         vq->used->ring[cur_idx & (vq->size - 1)].id
 317                                 = vq->buf_vec[vec_idx].desc_idx;
 318                         vq->used->ring[cur_idx & (vq->size - 1)].len
 319                                 = entry_len;
 320
 321                         entry_len = 0;
 322                         cur_idx++;
 323                         entry_success++;
 324                 }
 325
 326                 vec_idx++;
 327                 vb_addr = gpa_to_vva(dev, vq->buf_vec[vec_idx].buf_addr);
 328
 329                 /* Prefetch buffer address. */
 330                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 331                 vb_offset = 0;
 332                 vb_avail = vq->buf_vec[vec_idx].buf_len;
 333         }
 334
 335         cpy_len = RTE_MIN(vb_avail, seg_avail);
 336
 337         while (cpy_len > 0) {
 338                 /* Copy mbuf data to vring buffer */
 339                 rte_memcpy((void *)(uintptr_t)(vb_addr + vb_offset),
 340                         rte_pktmbuf_mtod_offset(pkt, const void *, seg_offset),
 341                         cpy_len);
 342
 343                 PRINT_PACKET(dev,
 344                         (uintptr_t)(vb_addr + vb_offset),
 345                         cpy_len, 0);
 346
 347                 seg_offset += cpy_len;
 348                 vb_offset += cpy_len;
 349                 seg_avail -= cpy_len;
 350                 vb_avail -= cpy_len;
 351                 entry_len += cpy_len;
 352
 353                 if (seg_avail != 0) {
 354                         /*
 355                          * The virtio buffer in this vring
 356                          * entry reach to its end.
 357                          * But the segment doesn't complete.
 358                          */
 359                         if ((vq->desc[vq->buf_vec[vec_idx].desc_idx].flags &
 360                                 VRING_DESC_F_NEXT) == 0) {
 361                                 /* Update used ring with desc information */
 362                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 363                                         = vq->buf_vec[vec_idx].desc_idx;
 364                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 365                                         = entry_len;
 366                                 entry_len = 0;
 367                                 cur_idx++;
 368                                 entry_success++;
 369                         }
 370
 371                         vec_idx++;
 372                         vb_addr = gpa_to_vva(dev,
 373                                 vq->buf_vec[vec_idx].buf_addr);
 374                         vb_offset = 0;
 375                         vb_avail = vq->buf_vec[vec_idx].buf_len;
 376                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 377                 } else {
 378                         /*
 379                          * This current segment complete, need continue to
 380                          * check if the whole packet complete or not.
 381                          */
 382                         pkt = pkt->next;
 383                         if (pkt != NULL) {
 384                                 /*
 385                                  * There are more segments.
 386                                  */
 387                                 if (vb_avail == 0) {
 388                                         /*
 389                                          * This current buffer from vring is
 390                                          * used up, need fetch next buffer
 391                                          * from buf_vec.
 392                                          */
 393                                         uint32_t desc_idx =
 394                                                 vq->buf_vec[vec_idx].desc_idx;
 395
 396                                         if ((vq->desc[desc_idx].flags &
 397                                                 VRING_DESC_F_NEXT) == 0) {
 398                                                 uint16_t wrapped_idx =
 399                                                         cur_idx & (vq->size - 1);
 400                                                 /*
 401                                                  * Update used ring with the
 402                                                  * descriptor information
 403                                                  */
 404                                                 vq->used->ring[wrapped_idx].id
 405                                                         = desc_idx;
 406                                                 vq->used->ring[wrapped_idx].len
 407                                                         = entry_len;
 408                                                 entry_success++;
 409                                                 entry_len = 0;
 410                                                 cur_idx++;
 411                                         }
 412
 413                                         /* Get next buffer from buf_vec. */
 414                                         vec_idx++;
 415                                         vb_addr = gpa_to_vva(dev,
 416                                                 vq->buf_vec[vec_idx].buf_addr);
 417                                         vb_avail =
 418                                                 vq->buf_vec[vec_idx].buf_len;
 419                                         vb_offset = 0;
 420                                 }
 421
 422                                 seg_offset = 0;
 423                                 seg_avail = rte_pktmbuf_data_len(pkt);
 424                                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 425                         } else {
 426                                 /*
 427                                  * This whole packet completes.
 428                                  */
 429                                 /* Update used ring with desc information */
 430                                 vq->used->ring[cur_idx & (vq->size - 1)].id
 431                                         = vq->buf_vec[vec_idx].desc_idx;
 432                                 vq->used->ring[cur_idx & (vq->size - 1)].len
 433                                         = entry_len;
 434                                 entry_success++;
 435                                 break;
 436                         }
 437                 }
 438         }
 439
 440         return entry_success;
 441 }
 442
 443 static inline void __attribute__((always_inline))
 444 update_secure_len(struct vhost_virtqueue *vq, uint32_t id,
 445         uint32_t *secure_len, uint32_t *vec_idx)
 446 {
 447         uint16_t wrapped_idx = id & (vq->size - 1);
 448         uint32_t idx = vq->avail->ring[wrapped_idx];
 449         uint8_t next_desc;
 450         uint32_t len = *secure_len;
 451         uint32_t vec_id = *vec_idx;
 452
 453         do {
 454                 next_desc = 0;
 455                 len += vq->desc[idx].len;
 456                 vq->buf_vec[vec_id].buf_addr = vq->desc[idx].addr;
 457                 vq->buf_vec[vec_id].buf_len = vq->desc[idx].len;
 458                 vq->buf_vec[vec_id].desc_idx = idx;
 459                 vec_id++;
 460
 461                 if (vq->desc[idx].flags & VRING_DESC_F_NEXT) {
 462                         idx = vq->desc[idx].next;
 463                         next_desc = 1;
 464                 }
 465         } while (next_desc);
 466
 467         *secure_len = len;
 468         *vec_idx = vec_id;
 469 }
 470
 471 /*
 472  * This function works for mergeable RX.
 473  */
 474 static inline uint32_t __attribute__((always_inline))
 475 virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 476         struct rte_mbuf **pkts, uint32_t count)
 477 {
 478         struct vhost_virtqueue *vq;
 479         uint32_t pkt_idx = 0, entry_success = 0;
 480         uint16_t avail_idx;
 481         uint16_t res_base_idx, res_cur_idx;
 482         uint8_t success = 0;
 483
 484         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_merge_rx()\n",
 485                 dev->device_fh);
 486         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) {
 487                 RTE_LOG(ERR, VHOST_DATA,
 488                         "%s (%"PRIu64"): virtqueue idx:%d invalid.\n",
 489                         __func__, dev->device_fh, queue_id);
 490                 return 0;
 491         }
 492
 493         vq = dev->virtqueue[queue_id];
 494         if (unlikely(vq->enabled == 0))
 495                 return 0;
 496
 497         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
 498
 499         if (count == 0)
 500                 return 0;
 501
 502         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 503                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vhost_hlen;
 504
 505                 do {
 506                         /*
 507                          * As many data cores may want access to available
 508                          * buffers, they need to be reserved.
 509                          */
 510                         uint32_t secure_len = 0;
 511                         uint32_t vec_idx = 0;
 512
 513                         res_base_idx = vq->last_used_idx_res;
 514                         res_cur_idx = res_base_idx;
 515
 516                         do {
 517                                 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 518                                 if (unlikely(res_cur_idx == avail_idx))
 519                                         goto merge_rx_exit;
 520
 521                                 update_secure_len(vq, res_cur_idx,
 522                                                   &secure_len, &vec_idx);
 523                                 res_cur_idx++;
 524                         } while (pkt_len > secure_len);
 525
 526                         /* vq->last_used_idx_res is atomically updated. */
 527                         success = rte_atomic16_cmpset(&vq->last_used_idx_res,
 528                                                         res_base_idx,
 529                                                         res_cur_idx);
 530                 } while (success == 0);
 531
 532                 entry_success = copy_from_mbuf_to_vring(dev, queue_id,
 533                         res_base_idx, res_cur_idx, pkts[pkt_idx]);
 534
 535                 rte_compiler_barrier();
 536
 537                 /*
 538                  * Wait until it's our turn to add our buffer
 539                  * to the used ring.
 540                  */
 541                 while (unlikely(vq->last_used_idx != res_base_idx))
 542                         rte_pause();
 543
 544                 *(volatile uint16_t *)&vq->used->idx += entry_success;
 545                 vq->last_used_idx = res_cur_idx;
 546         }
 547
 548 merge_rx_exit:
 549         if (likely(pkt_idx)) {
 550                 /* flush used->idx update before we read avail->flags. */
 551                 rte_mb();
 552
 553                 /* Kick the guest if necessary. */
 554                 if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 555                         eventfd_write(vq->callfd, (eventfd_t)1);
 556         }
 557
 558         return pkt_idx;
 559 }
 560
 561 uint16_t
 562 rte_vhost_enqueue_burst(struct virtio_net *dev, uint16_t queue_id,
 563         struct rte_mbuf **pkts, uint16_t count)
 564 {
 565         if (unlikely(dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)))
 566                 return virtio_dev_merge_rx(dev, queue_id, pkts, count);
 567         else
 568                 return virtio_dev_rx(dev, queue_id, pkts, count);
 569 }
 570
 571 static void
 572 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
 573 {
 574         struct ipv4_hdr *ipv4_hdr;
 575         struct ipv6_hdr *ipv6_hdr;
 576         void *l3_hdr = NULL;
 577         struct ether_hdr *eth_hdr;
 578         uint16_t ethertype;
 579
 580         eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 581
 582         m->l2_len = sizeof(struct ether_hdr);
 583         ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
 584
 585         if (ethertype == ETHER_TYPE_VLAN) {
 586                 struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
 587
 588                 m->l2_len += sizeof(struct vlan_hdr);
 589                 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
 590         }
 591
 592         l3_hdr = (char *)eth_hdr + m->l2_len;
 593
 594         switch (ethertype) {
 595         case ETHER_TYPE_IPv4:
 596                 ipv4_hdr = (struct ipv4_hdr *)l3_hdr;
 597                 *l4_proto = ipv4_hdr->next_proto_id;
 598                 m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
 599                 *l4_hdr = (char *)l3_hdr + m->l3_len;
 600                 m->ol_flags |= PKT_TX_IPV4;
 601                 break;
 602         case ETHER_TYPE_IPv6:
 603                 ipv6_hdr = (struct ipv6_hdr *)l3_hdr;
 604                 *l4_proto = ipv6_hdr->proto;
 605                 m->l3_len = sizeof(struct ipv6_hdr);
 606                 *l4_hdr = (char *)l3_hdr + m->l3_len;
 607                 m->ol_flags |= PKT_TX_IPV6;
 608                 break;
 609         default:
 610                 m->l3_len = 0;
 611                 *l4_proto = 0;
 612                 break;
 613         }
 614 }
 615
 616 static inline void __attribute__((always_inline))
 617 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
 618 {
 619         uint16_t l4_proto = 0;
 620         void *l4_hdr = NULL;
 621         struct tcp_hdr *tcp_hdr = NULL;
 622
 623         parse_ethernet(m, &l4_proto, &l4_hdr);
 624         if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
 625                 if (hdr->csum_start == (m->l2_len + m->l3_len)) {
 626                         switch (hdr->csum_offset) {
 627                         case (offsetof(struct tcp_hdr, cksum)):
 628                                 if (l4_proto == IPPROTO_TCP)
 629                                         m->ol_flags |= PKT_TX_TCP_CKSUM;
 630                                 break;
 631                         case (offsetof(struct udp_hdr, dgram_cksum)):
 632                                 if (l4_proto == IPPROTO_UDP)
 633                                         m->ol_flags |= PKT_TX_UDP_CKSUM;
 634                                 break;
 635                         case (offsetof(struct sctp_hdr, cksum)):
 636                                 if (l4_proto == IPPROTO_SCTP)
 637                                         m->ol_flags |= PKT_TX_SCTP_CKSUM;
 638                                 break;
 639                         default:
 640                                 break;
 641                         }
 642                 }
 643         }
 644
 645         if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
 646                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
 647                 case VIRTIO_NET_HDR_GSO_TCPV4:
 648                 case VIRTIO_NET_HDR_GSO_TCPV6:
 649                         tcp_hdr = (struct tcp_hdr *)l4_hdr;
 650                         m->ol_flags |= PKT_TX_TCP_SEG;
 651                         m->tso_segsz = hdr->gso_size;
 652                         m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
 653                         break;
 654                 default:
 655                         RTE_LOG(WARNING, VHOST_DATA,
 656                                 "unsupported gso type %u.\n", hdr->gso_type);
 657                         break;
 658                 }
 659         }
 660 }
 661
 662 uint16_t
 663 rte_vhost_dequeue_burst(struct virtio_net *dev, uint16_t queue_id,
 664         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 665 {
 666         struct rte_mbuf *m, *prev;
 667         struct vhost_virtqueue *vq;
 668         struct vring_desc *desc;
 669         uint64_t vb_addr = 0;
 670         uint64_t vb_net_hdr_addr = 0;
 671         uint32_t head[MAX_PKT_BURST];
 672         uint32_t used_idx;
 673         uint32_t i;
 674         uint16_t free_entries, entry_success = 0;
 675         uint16_t avail_idx;
 676         struct virtio_net_hdr *hdr = NULL;
 677
 678         if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->virt_qp_nb))) {
 679                 RTE_LOG(ERR, VHOST_DATA,
 680                         "%s (%"PRIu64"): virtqueue idx:%d invalid.\n",
 681                         __func__, dev->device_fh, queue_id);
 682                 return 0;
 683         }
 684
 685         vq = dev->virtqueue[queue_id];
 686         if (unlikely(vq->enabled == 0))
 687                 return 0;
 688
 689         avail_idx =  *((volatile uint16_t *)&vq->avail->idx);
 690
 691         /* If there are no available buffers then return. */
 692         if (vq->last_used_idx == avail_idx)
 693                 return 0;
 694
 695         LOG_DEBUG(VHOST_DATA, "%s (%"PRIu64")\n", __func__,
 696                 dev->device_fh);
 697
 698         /* Prefetch available ring to retrieve head indexes. */
 699         rte_prefetch0(&vq->avail->ring[vq->last_used_idx & (vq->size - 1)]);
 700
 701         /*get the number of free entries in the ring*/
 702         free_entries = (avail_idx - vq->last_used_idx);
 703
 704         free_entries = RTE_MIN(free_entries, count);
 705         /* Limit to MAX_PKT_BURST. */
 706         free_entries = RTE_MIN(free_entries, MAX_PKT_BURST);
 707
 708         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n",
 709                         dev->device_fh, free_entries);
 710         /* Retrieve all of the head indexes first to avoid caching issues. */
 711         for (i = 0; i < free_entries; i++)
 712                 head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)];
 713
 714         /* Prefetch descriptor index. */
 715         rte_prefetch0(&vq->desc[head[entry_success]]);
 716         rte_prefetch0(&vq->used->ring[vq->last_used_idx & (vq->size - 1)]);
 717
 718         while (entry_success < free_entries) {
 719                 uint32_t vb_avail, vb_offset;
 720                 uint32_t seg_avail, seg_offset;
 721                 uint32_t cpy_len;
 722                 uint32_t seg_num = 0;
 723                 struct rte_mbuf *cur;
 724                 uint8_t alloc_err = 0;
 725
 726                 desc = &vq->desc[head[entry_success]];
 727
 728                 vb_net_hdr_addr = gpa_to_vva(dev, desc->addr);
 729                 hdr = (struct virtio_net_hdr *)((uintptr_t)vb_net_hdr_addr);
 730
 731                 /* Discard first buffer as it is the virtio header */
 732                 if (desc->flags & VRING_DESC_F_NEXT) {
 733                         desc = &vq->desc[desc->next];
 734                         vb_offset = 0;
 735                         vb_avail = desc->len;
 736                 } else {
 737                         vb_offset = vq->vhost_hlen;
 738                         vb_avail = desc->len - vb_offset;
 739                 }
 740
 741                 /* Buffer address translation. */
 742                 vb_addr = gpa_to_vva(dev, desc->addr);
 743                 /* Prefetch buffer address. */
 744                 rte_prefetch0((void *)(uintptr_t)vb_addr);
 745
 746                 used_idx = vq->last_used_idx & (vq->size - 1);
 747
 748                 if (entry_success < (free_entries - 1)) {
 749                         /* Prefetch descriptor index. */
 750                         rte_prefetch0(&vq->desc[head[entry_success+1]]);
 751                         rte_prefetch0(&vq->used->ring[(used_idx + 1) & (vq->size - 1)]);
 752                 }
 753
 754                 /* Update used index buffer information. */
 755                 vq->used->ring[used_idx].id = head[entry_success];
 756                 vq->used->ring[used_idx].len = 0;
 757
 758                 /* Allocate an mbuf and populate the structure. */
 759                 m = rte_pktmbuf_alloc(mbuf_pool);
 760                 if (unlikely(m == NULL)) {
 761                         RTE_LOG(ERR, VHOST_DATA,
 762                                 "Failed to allocate memory for mbuf.\n");
 763                         break;
 764                 }
 765                 seg_offset = 0;
 766                 seg_avail = m->buf_len - RTE_PKTMBUF_HEADROOM;
 767                 cpy_len = RTE_MIN(vb_avail, seg_avail);
 768
 769                 PRINT_PACKET(dev, (uintptr_t)vb_addr, desc->len, 0);
 770
 771                 seg_num++;
 772                 cur = m;
 773                 prev = m;
 774                 while (cpy_len != 0) {
 775                         rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *, seg_offset),
 776                                 (void *)((uintptr_t)(vb_addr + vb_offset)),
 777                                 cpy_len);
 778
 779                         seg_offset += cpy_len;
 780                         vb_offset += cpy_len;
 781                         vb_avail -= cpy_len;
 782                         seg_avail -= cpy_len;
 783
 784                         if (vb_avail != 0) {
 785                                 /*
 786                                  * The segment reachs to its end,
 787                                  * while the virtio buffer in TX vring has
 788                                  * more data to be copied.
 789                                  */
 790                                 cur->data_len = seg_offset;
 791                                 m->pkt_len += seg_offset;
 792                                 /* Allocate mbuf and populate the structure. */
 793                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 794                                 if (unlikely(cur == NULL)) {
 795                                         RTE_LOG(ERR, VHOST_DATA, "Failed to "
 796                                                 "allocate memory for mbuf.\n");
 797                                         rte_pktmbuf_free(m);
 798                                         alloc_err = 1;
 799                                         break;
 800                                 }
 801
 802                                 seg_num++;
 803                                 prev->next = cur;
 804                                 prev = cur;
 805                                 seg_offset = 0;
 806                                 seg_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM;
 807                         } else {
 808                                 if (desc->flags & VRING_DESC_F_NEXT) {
 809                                         /*
 810                                          * There are more virtio buffers in
 811                                          * same vring entry need to be copied.
 812                                          */
 813                                         if (seg_avail == 0) {
 814                                                 /*
 815                                                  * The current segment hasn't
 816                                                  * room to accomodate more
 817                                                  * data.
 818                                                  */
 819                                                 cur->data_len = seg_offset;
 820                                                 m->pkt_len += seg_offset;
 821                                                 /*
 822                                                  * Allocate an mbuf and
 823                                                  * populate the structure.
 824                                                  */
 825                                                 cur = rte_pktmbuf_alloc(mbuf_pool);
 826                                                 if (unlikely(cur == NULL)) {
 827                                                         RTE_LOG(ERR,
 828                                                                 VHOST_DATA,
 829                                                                 "Failed to "
 830                                                                 "allocate memory "
 831                                                                 "for mbuf\n");
 832                                                         rte_pktmbuf_free(m);
 833                                                         alloc_err = 1;
 834                                                         break;
 835                                                 }
 836                                                 seg_num++;
 837                                                 prev->next = cur;
 838                                                 prev = cur;
 839                                                 seg_offset = 0;
 840                                                 seg_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM;
 841                                         }
 842
 843                                         desc = &vq->desc[desc->next];
 844
 845                                         /* Buffer address translation. */
 846                                         vb_addr = gpa_to_vva(dev, desc->addr);
 847                                         /* Prefetch buffer address. */
 848                                         rte_prefetch0((void *)(uintptr_t)vb_addr);
 849                                         vb_offset = 0;
 850                                         vb_avail = desc->len;
 851
 852                                         PRINT_PACKET(dev, (uintptr_t)vb_addr,
 853                                                 desc->len, 0);
 854                                 } else {
 855                                         /* The whole packet completes. */
 856                                         cur->data_len = seg_offset;
 857                                         m->pkt_len += seg_offset;
 858                                         vb_avail = 0;
 859                                 }
 860                         }
 861
 862                         cpy_len = RTE_MIN(vb_avail, seg_avail);
 863                 }
 864
 865                 if (unlikely(alloc_err == 1))
 866                         break;
 867
 868                 m->nb_segs = seg_num;
 869                 if ((hdr->flags != 0) || (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE))
 870                         vhost_dequeue_offload(hdr, m);
 871
 872                 pkts[entry_success] = m;
 873                 vq->last_used_idx++;
 874                 entry_success++;
 875         }
 876
 877         rte_compiler_barrier();
 878         vq->used->idx += entry_success;
 879         /* Kick guest if required. */
 880         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
 881                 eventfd_write(vq->callfd, (eventfd_t)1);
 882         return entry_success;
 883 }