lib/librte_vhost/virtio_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdbool.h>
   7 #include <linux/virtio_net.h>
   8
   9 #include <rte_mbuf.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_ether.h>
  12 #include <rte_ip.h>
  13 #include <rte_vhost.h>
  14 #include <rte_tcp.h>
  15 #include <rte_udp.h>
  16 #include <rte_sctp.h>
  17 #include <rte_arp.h>
  18 #include <rte_spinlock.h>
  19 #include <rte_malloc.h>
  20
  21 #include "iotlb.h"
  22 #include "vhost.h"
  23
  24 #define MAX_PKT_BURST 32
  25
  26 #define MAX_BATCH_LEN 256
  27
  28 static  __rte_always_inline bool
  29 rxvq_is_mergeable(struct virtio_net *dev)
  30 {
  31         return dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF);
  32 }
  33
  34 static bool
  35 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
  36 {
  37         return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring;
  38 }
  39
  40 static __rte_always_inline void
  41 do_flush_shadow_used_ring_split(struct virtio_net *dev,
  42                         struct vhost_virtqueue *vq,
  43                         uint16_t to, uint16_t from, uint16_t size)
  44 {
  45         rte_memcpy(&vq->used->ring[to],
  46                         &vq->shadow_used_split[from],
  47                         size * sizeof(struct vring_used_elem));
  48         vhost_log_cache_used_vring(dev, vq,
  49                         offsetof(struct vring_used, ring[to]),
  50                         size * sizeof(struct vring_used_elem));
  51 }
  52
  53 static __rte_always_inline void
  54 flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
  55 {
  56         uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
  57
  58         if (used_idx + vq->shadow_used_idx <= vq->size) {
  59                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0,
  60                                           vq->shadow_used_idx);
  61         } else {
  62                 uint16_t size;
  63
  64                 /* update used ring interval [used_idx, vq->size] */
  65                 size = vq->size - used_idx;
  66                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size);
  67
  68                 /* update the left half used ring interval [0, left_size] */
  69                 do_flush_shadow_used_ring_split(dev, vq, 0, size,
  70                                           vq->shadow_used_idx - size);
  71         }
  72         vq->last_used_idx += vq->shadow_used_idx;
  73
  74         rte_smp_wmb();
  75
  76         vhost_log_cache_sync(dev, vq);
  77
  78         *(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx;
  79         vq->shadow_used_idx = 0;
  80         vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
  81                 sizeof(vq->used->idx));
  82 }
  83
  84 static __rte_always_inline void
  85 update_shadow_used_ring_split(struct vhost_virtqueue *vq,
  86                          uint16_t desc_idx, uint32_t len)
  87 {
  88         uint16_t i = vq->shadow_used_idx++;
  89
  90         vq->shadow_used_split[i].id  = desc_idx;
  91         vq->shadow_used_split[i].len = len;
  92 }
  93
  94 static __rte_always_inline void
  95 flush_shadow_used_ring_packed(struct virtio_net *dev,
  96                         struct vhost_virtqueue *vq)
  97 {
  98         int i;
  99         uint16_t used_idx = vq->last_used_idx;
 100         uint16_t head_idx = vq->last_used_idx;
 101         uint16_t head_flags = 0;
 102
 103         /* Split loop in two to save memory barriers */
 104         for (i = 0; i < vq->shadow_used_idx; i++) {
 105                 vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id;
 106                 vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len;
 107
 108                 used_idx += vq->shadow_used_packed[i].count;
 109                 if (used_idx >= vq->size)
 110                         used_idx -= vq->size;
 111         }
 112
 113         for (i = 0; i < vq->shadow_used_idx; i++) {
 114                 uint16_t flags;
 115
 116                 if (vq->shadow_used_packed[i].len)
 117                         flags = VRING_DESC_F_WRITE;
 118                 else
 119                         flags = 0;
 120
 121                 if (vq->used_wrap_counter) {
 122                         flags |= VRING_DESC_F_USED;
 123                         flags |= VRING_DESC_F_AVAIL;
 124                 } else {
 125                         flags &= ~VRING_DESC_F_USED;
 126                         flags &= ~VRING_DESC_F_AVAIL;
 127                 }
 128
 129                 if (i > 0) {
 130                         vq->desc_packed[vq->last_used_idx].flags = flags;
 131
 132                         vhost_log_cache_used_vring(dev, vq,
 133                                         vq->last_used_idx *
 134                                         sizeof(struct vring_packed_desc),
 135                                         sizeof(struct vring_packed_desc));
 136                 } else {
 137                         head_idx = vq->last_used_idx;
 138                         head_flags = flags;
 139                 }
 140
 141                 vq->last_used_idx += vq->shadow_used_packed[i].count;
 142                 if (vq->last_used_idx >= vq->size) {
 143                         vq->used_wrap_counter ^= 1;
 144                         vq->last_used_idx -= vq->size;
 145                 }
 146         }
 147
 148         __atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
 149                          __ATOMIC_RELEASE);
 150
 151         vhost_log_cache_used_vring(dev, vq,
 152                                 head_idx *
 153                                 sizeof(struct vring_packed_desc),
 154                                 sizeof(struct vring_packed_desc));
 155
 156         vq->shadow_used_idx = 0;
 157         vhost_log_cache_sync(dev, vq);
 158 }
 159
 160 static __rte_always_inline void
 161 update_shadow_used_ring_packed(struct vhost_virtqueue *vq,
 162                          uint16_t desc_idx, uint32_t len, uint16_t count)
 163 {
 164         uint16_t i = vq->shadow_used_idx++;
 165
 166         vq->shadow_used_packed[i].id  = desc_idx;
 167         vq->shadow_used_packed[i].len = len;
 168         vq->shadow_used_packed[i].count = count;
 169 }
 170
 171 static inline void
 172 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
 173 {
 174         struct batch_copy_elem *elem = vq->batch_copy_elems;
 175         uint16_t count = vq->batch_copy_nb_elems;
 176         int i;
 177
 178         for (i = 0; i < count; i++) {
 179                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
 180                 vhost_log_cache_write(dev, vq, elem[i].log_addr, elem[i].len);
 181                 PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
 182         }
 183
 184         vq->batch_copy_nb_elems = 0;
 185 }
 186
 187 static inline void
 188 do_data_copy_dequeue(struct vhost_virtqueue *vq)
 189 {
 190         struct batch_copy_elem *elem = vq->batch_copy_elems;
 191         uint16_t count = vq->batch_copy_nb_elems;
 192         int i;
 193
 194         for (i = 0; i < count; i++)
 195                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
 196
 197         vq->batch_copy_nb_elems = 0;
 198 }
 199
 200 /* avoid write operation when necessary, to lessen cache issues */
 201 #define ASSIGN_UNLESS_EQUAL(var, val) do {      \
 202         if ((var) != (val))                     \
 203                 (var) = (val);                  \
 204 } while (0)
 205
 206 static __rte_always_inline void
 207 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 208 {
 209         uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK;
 210
 211         if (m_buf->ol_flags & PKT_TX_TCP_SEG)
 212                 csum_l4 |= PKT_TX_TCP_CKSUM;
 213
 214         if (csum_l4) {
 215                 net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
 216                 net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len;
 217
 218                 switch (csum_l4) {
 219                 case PKT_TX_TCP_CKSUM:
 220                         net_hdr->csum_offset = (offsetof(struct rte_tcp_hdr,
 221                                                 cksum));
 222                         break;
 223                 case PKT_TX_UDP_CKSUM:
 224                         net_hdr->csum_offset = (offsetof(struct rte_udp_hdr,
 225                                                 dgram_cksum));
 226                         break;
 227                 case PKT_TX_SCTP_CKSUM:
 228                         net_hdr->csum_offset = (offsetof(struct rte_sctp_hdr,
 229                                                 cksum));
 230                         break;
 231                 }
 232         } else {
 233                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0);
 234                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0);
 235                 ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0);
 236         }
 237
 238         /* IP cksum verification cannot be bypassed, then calculate here */
 239         if (m_buf->ol_flags & PKT_TX_IP_CKSUM) {
 240                 struct rte_ipv4_hdr *ipv4_hdr;
 241
 242                 ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct rte_ipv4_hdr *,
 243                                                    m_buf->l2_len);
 244                 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
 245         }
 246
 247         if (m_buf->ol_flags & PKT_TX_TCP_SEG) {
 248                 if (m_buf->ol_flags & PKT_TX_IPV4)
 249                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
 250                 else
 251                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
 252                 net_hdr->gso_size = m_buf->tso_segsz;
 253                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len
 254                                         + m_buf->l4_len;
 255         } else if (m_buf->ol_flags & PKT_TX_UDP_SEG) {
 256                 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
 257                 net_hdr->gso_size = m_buf->tso_segsz;
 258                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len +
 259                         m_buf->l4_len;
 260         } else {
 261                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0);
 262                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0);
 263                 ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0);
 264         }
 265 }
 266
 267 static __rte_always_inline int
 268 map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 269                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 270                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 271 {
 272         uint16_t vec_id = *vec_idx;
 273
 274         while (desc_len) {
 275                 uint64_t desc_addr;
 276                 uint64_t desc_chunck_len = desc_len;
 277
 278                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 279                         return -1;
 280
 281                 desc_addr = vhost_iova_to_vva(dev, vq,
 282                                 desc_iova,
 283                                 &desc_chunck_len,
 284                                 perm);
 285                 if (unlikely(!desc_addr))
 286                         return -1;
 287
 288                 rte_prefetch0((void *)(uintptr_t)desc_addr);
 289
 290                 buf_vec[vec_id].buf_iova = desc_iova;
 291                 buf_vec[vec_id].buf_addr = desc_addr;
 292                 buf_vec[vec_id].buf_len  = desc_chunck_len;
 293
 294                 desc_len -= desc_chunck_len;
 295                 desc_iova += desc_chunck_len;
 296                 vec_id++;
 297         }
 298         *vec_idx = vec_id;
 299
 300         return 0;
 301 }
 302
 303 static __rte_always_inline int
 304 fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 305                          uint32_t avail_idx, uint16_t *vec_idx,
 306                          struct buf_vector *buf_vec, uint16_t *desc_chain_head,
 307                          uint32_t *desc_chain_len, uint8_t perm)
 308 {
 309         uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
 310         uint16_t vec_id = *vec_idx;
 311         uint32_t len    = 0;
 312         uint64_t dlen;
 313         uint32_t nr_descs = vq->size;
 314         uint32_t cnt    = 0;
 315         struct vring_desc *descs = vq->desc;
 316         struct vring_desc *idesc = NULL;
 317
 318         if (unlikely(idx >= vq->size))
 319                 return -1;
 320
 321         *desc_chain_head = idx;
 322
 323         if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
 324                 dlen = vq->desc[idx].len;
 325                 nr_descs = dlen / sizeof(struct vring_desc);
 326                 if (unlikely(nr_descs > vq->size))
 327                         return -1;
 328
 329                 descs = (struct vring_desc *)(uintptr_t)
 330                         vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
 331                                                 &dlen,
 332                                                 VHOST_ACCESS_RO);
 333                 if (unlikely(!descs))
 334                         return -1;
 335
 336                 if (unlikely(dlen < vq->desc[idx].len)) {
 337                         /*
 338                          * The indirect desc table is not contiguous
 339                          * in process VA space, we have to copy it.
 340                          */
 341                         idesc = vhost_alloc_copy_ind_table(dev, vq,
 342                                         vq->desc[idx].addr, vq->desc[idx].len);
 343                         if (unlikely(!idesc))
 344                                 return -1;
 345
 346                         descs = idesc;
 347                 }
 348
 349                 idx = 0;
 350         }
 351
 352         while (1) {
 353                 if (unlikely(idx >= nr_descs || cnt++ >= nr_descs)) {
 354                         free_ind_table(idesc);
 355                         return -1;
 356                 }
 357
 358                 len += descs[idx].len;
 359
 360                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 361                                                 descs[idx].addr, descs[idx].len,
 362                                                 perm))) {
 363                         free_ind_table(idesc);
 364                         return -1;
 365                 }
 366
 367                 if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
 368                         break;
 369
 370                 idx = descs[idx].next;
 371         }
 372
 373         *desc_chain_len = len;
 374         *vec_idx = vec_id;
 375
 376         if (unlikely(!!idesc))
 377                 free_ind_table(idesc);
 378
 379         return 0;
 380 }
 381
 382 /*
 383  * Returns -1 on fail, 0 on success
 384  */
 385 static inline int
 386 reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 387                                 uint32_t size, struct buf_vector *buf_vec,
 388                                 uint16_t *num_buffers, uint16_t avail_head,
 389                                 uint16_t *nr_vec)
 390 {
 391         uint16_t cur_idx;
 392         uint16_t vec_idx = 0;
 393         uint16_t max_tries, tries = 0;
 394
 395         uint16_t head_idx = 0;
 396         uint32_t len = 0;
 397
 398         *num_buffers = 0;
 399         cur_idx  = vq->last_avail_idx;
 400
 401         if (rxvq_is_mergeable(dev))
 402                 max_tries = vq->size - 1;
 403         else
 404                 max_tries = 1;
 405
 406         while (size > 0) {
 407                 if (unlikely(cur_idx == avail_head))
 408                         return -1;
 409                 /*
 410                  * if we tried all available ring items, and still
 411                  * can't get enough buf, it means something abnormal
 412                  * happened.
 413                  */
 414                 if (unlikely(++tries > max_tries))
 415                         return -1;
 416
 417                 if (unlikely(fill_vec_buf_split(dev, vq, cur_idx,
 418                                                 &vec_idx, buf_vec,
 419                                                 &head_idx, &len,
 420                                                 VHOST_ACCESS_RW) < 0))
 421                         return -1;
 422                 len = RTE_MIN(len, size);
 423                 update_shadow_used_ring_split(vq, head_idx, len);
 424                 size -= len;
 425
 426                 cur_idx++;
 427                 *num_buffers += 1;
 428         }
 429
 430         *nr_vec = vec_idx;
 431
 432         return 0;
 433 }
 434
 435 static __rte_always_inline int
 436 fill_vec_buf_packed_indirect(struct virtio_net *dev,
 437                         struct vhost_virtqueue *vq,
 438                         struct vring_packed_desc *desc, uint16_t *vec_idx,
 439                         struct buf_vector *buf_vec, uint32_t *len, uint8_t perm)
 440 {
 441         uint16_t i;
 442         uint32_t nr_descs;
 443         uint16_t vec_id = *vec_idx;
 444         uint64_t dlen;
 445         struct vring_packed_desc *descs, *idescs = NULL;
 446
 447         dlen = desc->len;
 448         descs = (struct vring_packed_desc *)(uintptr_t)
 449                 vhost_iova_to_vva(dev, vq, desc->addr, &dlen, VHOST_ACCESS_RO);
 450         if (unlikely(!descs))
 451                 return -1;
 452
 453         if (unlikely(dlen < desc->len)) {
 454                 /*
 455                  * The indirect desc table is not contiguous
 456                  * in process VA space, we have to copy it.
 457                  */
 458                 idescs = vhost_alloc_copy_ind_table(dev,
 459                                 vq, desc->addr, desc->len);
 460                 if (unlikely(!idescs))
 461                         return -1;
 462
 463                 descs = idescs;
 464         }
 465
 466         nr_descs =  desc->len / sizeof(struct vring_packed_desc);
 467         if (unlikely(nr_descs >= vq->size)) {
 468                 free_ind_table(idescs);
 469                 return -1;
 470         }
 471
 472         for (i = 0; i < nr_descs; i++) {
 473                 if (unlikely(vec_id >= BUF_VECTOR_MAX)) {
 474                         free_ind_table(idescs);
 475                         return -1;
 476                 }
 477
 478                 *len += descs[i].len;
 479                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 480                                                 descs[i].addr, descs[i].len,
 481                                                 perm)))
 482                         return -1;
 483         }
 484         *vec_idx = vec_id;
 485
 486         if (unlikely(!!idescs))
 487                 free_ind_table(idescs);
 488
 489         return 0;
 490 }
 491
 492 static __rte_always_inline int
 493 fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 494                                 uint16_t avail_idx, uint16_t *desc_count,
 495                                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 496                                 uint16_t *buf_id, uint32_t *len, uint8_t perm)
 497 {
 498         bool wrap_counter = vq->avail_wrap_counter;
 499         struct vring_packed_desc *descs = vq->desc_packed;
 500         uint16_t vec_id = *vec_idx;
 501
 502         if (avail_idx < vq->last_avail_idx)
 503                 wrap_counter ^= 1;
 504
 505         /*
 506          * Perform a load-acquire barrier in desc_is_avail to
 507          * enforce the ordering between desc flags and desc
 508          * content.
 509          */
 510         if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
 511                 return -1;
 512
 513         *desc_count = 0;
 514         *len = 0;
 515
 516         while (1) {
 517                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 518                         return -1;
 519
 520                 if (unlikely(*desc_count >= vq->size))
 521                         return -1;
 522
 523                 *desc_count += 1;
 524                 *buf_id = descs[avail_idx].id;
 525
 526                 if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) {
 527                         if (unlikely(fill_vec_buf_packed_indirect(dev, vq,
 528                                                         &descs[avail_idx],
 529                                                         &vec_id, buf_vec,
 530                                                         len, perm) < 0))
 531                                 return -1;
 532                 } else {
 533                         *len += descs[avail_idx].len;
 534
 535                         if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 536                                                         descs[avail_idx].addr,
 537                                                         descs[avail_idx].len,
 538                                                         perm)))
 539                                 return -1;
 540                 }
 541
 542                 if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0)
 543                         break;
 544
 545                 if (++avail_idx >= vq->size) {
 546                         avail_idx -= vq->size;
 547                         wrap_counter ^= 1;
 548                 }
 549         }
 550
 551         *vec_idx = vec_id;
 552
 553         return 0;
 554 }
 555
 556 /*
 557  * Returns -1 on fail, 0 on success
 558  */
 559 static inline int
 560 reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 561                                 uint32_t size, struct buf_vector *buf_vec,
 562                                 uint16_t *nr_vec, uint16_t *num_buffers,
 563                                 uint16_t *nr_descs)
 564 {
 565         uint16_t avail_idx;
 566         uint16_t vec_idx = 0;
 567         uint16_t max_tries, tries = 0;
 568
 569         uint16_t buf_id = 0;
 570         uint32_t len = 0;
 571         uint16_t desc_count;
 572
 573         *num_buffers = 0;
 574         avail_idx = vq->last_avail_idx;
 575
 576         if (rxvq_is_mergeable(dev))
 577                 max_tries = vq->size - 1;
 578         else
 579                 max_tries = 1;
 580
 581         while (size > 0) {
 582                 /*
 583                  * if we tried all available ring items, and still
 584                  * can't get enough buf, it means something abnormal
 585                  * happened.
 586                  */
 587                 if (unlikely(++tries > max_tries))
 588                         return -1;
 589
 590                 if (unlikely(fill_vec_buf_packed(dev, vq,
 591                                                 avail_idx, &desc_count,
 592                                                 buf_vec, &vec_idx,
 593                                                 &buf_id, &len,
 594                                                 VHOST_ACCESS_RW) < 0))
 595                         return -1;
 596
 597                 len = RTE_MIN(len, size);
 598                 update_shadow_used_ring_packed(vq, buf_id, len, desc_count);
 599                 size -= len;
 600
 601                 avail_idx += desc_count;
 602                 if (avail_idx >= vq->size)
 603                         avail_idx -= vq->size;
 604
 605                 *nr_descs += desc_count;
 606                 *num_buffers += 1;
 607         }
 608
 609         *nr_vec = vec_idx;
 610
 611         return 0;
 612 }
 613
 614 static __rte_noinline void
 615 copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 616                 struct buf_vector *buf_vec,
 617                 struct virtio_net_hdr_mrg_rxbuf *hdr)
 618 {
 619         uint64_t len;
 620         uint64_t remain = dev->vhost_hlen;
 621         uint64_t src = (uint64_t)(uintptr_t)hdr, dst;
 622         uint64_t iova = buf_vec->buf_iova;
 623
 624         while (remain) {
 625                 len = RTE_MIN(remain,
 626                                 buf_vec->buf_len);
 627                 dst = buf_vec->buf_addr;
 628                 rte_memcpy((void *)(uintptr_t)dst,
 629                                 (void *)(uintptr_t)src,
 630                                 len);
 631
 632                 PRINT_PACKET(dev, (uintptr_t)dst,
 633                                 (uint32_t)len, 0);
 634                 vhost_log_cache_write(dev, vq,
 635                                 iova, len);
 636
 637                 remain -= len;
 638                 iova += len;
 639                 src += len;
 640                 buf_vec++;
 641         }
 642 }
 643
 644 static __rte_always_inline int
 645 copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 646                             struct rte_mbuf *m, struct buf_vector *buf_vec,
 647                             uint16_t nr_vec, uint16_t num_buffers)
 648 {
 649         uint32_t vec_idx = 0;
 650         uint32_t mbuf_offset, mbuf_avail;
 651         uint32_t buf_offset, buf_avail;
 652         uint64_t buf_addr, buf_iova, buf_len;
 653         uint32_t cpy_len;
 654         uint64_t hdr_addr;
 655         struct rte_mbuf *hdr_mbuf;
 656         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 657         struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
 658         int error = 0;
 659
 660         if (unlikely(m == NULL)) {
 661                 error = -1;
 662                 goto out;
 663         }
 664
 665         buf_addr = buf_vec[vec_idx].buf_addr;
 666         buf_iova = buf_vec[vec_idx].buf_iova;
 667         buf_len = buf_vec[vec_idx].buf_len;
 668
 669         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 670                 error = -1;
 671                 goto out;
 672         }
 673
 674         hdr_mbuf = m;
 675         hdr_addr = buf_addr;
 676         if (unlikely(buf_len < dev->vhost_hlen))
 677                 hdr = &tmp_hdr;
 678         else
 679                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
 680
 681         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
 682                 dev->vid, num_buffers);
 683
 684         if (unlikely(buf_len < dev->vhost_hlen)) {
 685                 buf_offset = dev->vhost_hlen - buf_len;
 686                 vec_idx++;
 687                 buf_addr = buf_vec[vec_idx].buf_addr;
 688                 buf_iova = buf_vec[vec_idx].buf_iova;
 689                 buf_len = buf_vec[vec_idx].buf_len;
 690                 buf_avail = buf_len - buf_offset;
 691         } else {
 692                 buf_offset = dev->vhost_hlen;
 693                 buf_avail = buf_len - dev->vhost_hlen;
 694         }
 695
 696         mbuf_avail  = rte_pktmbuf_data_len(m);
 697         mbuf_offset = 0;
 698         while (mbuf_avail != 0 || m->next != NULL) {
 699                 /* done with current buf, get the next one */
 700                 if (buf_avail == 0) {
 701                         vec_idx++;
 702                         if (unlikely(vec_idx >= nr_vec)) {
 703                                 error = -1;
 704                                 goto out;
 705                         }
 706
 707                         buf_addr = buf_vec[vec_idx].buf_addr;
 708                         buf_iova = buf_vec[vec_idx].buf_iova;
 709                         buf_len = buf_vec[vec_idx].buf_len;
 710
 711                         buf_offset = 0;
 712                         buf_avail  = buf_len;
 713                 }
 714
 715                 /* done with current mbuf, get the next one */
 716                 if (mbuf_avail == 0) {
 717                         m = m->next;
 718
 719                         mbuf_offset = 0;
 720                         mbuf_avail  = rte_pktmbuf_data_len(m);
 721                 }
 722
 723                 if (hdr_addr) {
 724                         virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
 725                         if (rxvq_is_mergeable(dev))
 726                                 ASSIGN_UNLESS_EQUAL(hdr->num_buffers,
 727                                                 num_buffers);
 728
 729                         if (unlikely(hdr == &tmp_hdr)) {
 730                                 copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr);
 731                         } else {
 732                                 PRINT_PACKET(dev, (uintptr_t)hdr_addr,
 733                                                 dev->vhost_hlen, 0);
 734                                 vhost_log_cache_write(dev, vq,
 735                                                 buf_vec[0].buf_iova,
 736                                                 dev->vhost_hlen);
 737                         }
 738
 739                         hdr_addr = 0;
 740                 }
 741
 742                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 743
 744                 if (likely(cpy_len > MAX_BATCH_LEN ||
 745                                         vq->batch_copy_nb_elems >= vq->size)) {
 746                         rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),
 747                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 748                                 cpy_len);
 749                         vhost_log_cache_write(dev, vq, buf_iova + buf_offset,
 750                                         cpy_len);
 751                         PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),
 752                                 cpy_len, 0);
 753                 } else {
 754                         batch_copy[vq->batch_copy_nb_elems].dst =
 755                                 (void *)((uintptr_t)(buf_addr + buf_offset));
 756                         batch_copy[vq->batch_copy_nb_elems].src =
 757                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
 758                         batch_copy[vq->batch_copy_nb_elems].log_addr =
 759                                 buf_iova + buf_offset;
 760                         batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 761                         vq->batch_copy_nb_elems++;
 762                 }
 763
 764                 mbuf_avail  -= cpy_len;
 765                 mbuf_offset += cpy_len;
 766                 buf_avail  -= cpy_len;
 767                 buf_offset += cpy_len;
 768         }
 769
 770 out:
 771
 772         return error;
 773 }
 774
 775 static __rte_noinline uint32_t
 776 virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 777         struct rte_mbuf **pkts, uint32_t count)
 778 {
 779         uint32_t pkt_idx = 0;
 780         uint16_t num_buffers;
 781         struct buf_vector buf_vec[BUF_VECTOR_MAX];
 782         uint16_t avail_head;
 783
 784         avail_head = *((volatile uint16_t *)&vq->avail->idx);
 785
 786         /*
 787          * The ordering between avail index and
 788          * desc reads needs to be enforced.
 789          */
 790         rte_smp_rmb();
 791
 792         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
 793
 794         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 795                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
 796                 uint16_t nr_vec = 0;
 797
 798                 if (unlikely(reserve_avail_buf_split(dev, vq,
 799                                                 pkt_len, buf_vec, &num_buffers,
 800                                                 avail_head, &nr_vec) < 0)) {
 801                         VHOST_LOG_DEBUG(VHOST_DATA,
 802                                 "(%d) failed to get enough desc from vring\n",
 803                                 dev->vid);
 804                         vq->shadow_used_idx -= num_buffers;
 805                         break;
 806                 }
 807
 808                 VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
 809                         dev->vid, vq->last_avail_idx,
 810                         vq->last_avail_idx + num_buffers);
 811
 812                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
 813                                                 buf_vec, nr_vec,
 814                                                 num_buffers) < 0) {
 815                         vq->shadow_used_idx -= num_buffers;
 816                         break;
 817                 }
 818
 819                 vq->last_avail_idx += num_buffers;
 820         }
 821
 822         do_data_copy_enqueue(dev, vq);
 823
 824         if (likely(vq->shadow_used_idx)) {
 825                 flush_shadow_used_ring_split(dev, vq);
 826                 vhost_vring_call_split(dev, vq);
 827         }
 828
 829         return pkt_idx;
 830 }
 831
 832 static __rte_noinline uint32_t
 833 virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 834         struct rte_mbuf **pkts, uint32_t count)
 835 {
 836         uint32_t pkt_idx = 0;
 837         uint16_t num_buffers;
 838         struct buf_vector buf_vec[BUF_VECTOR_MAX];
 839
 840         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 841                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
 842                 uint16_t nr_vec = 0;
 843                 uint16_t nr_descs = 0;
 844
 845                 if (unlikely(reserve_avail_buf_packed(dev, vq,
 846                                                 pkt_len, buf_vec, &nr_vec,
 847                                                 &num_buffers, &nr_descs) < 0)) {
 848                         VHOST_LOG_DEBUG(VHOST_DATA,
 849                                 "(%d) failed to get enough desc from vring\n",
 850                                 dev->vid);
 851                         vq->shadow_used_idx -= num_buffers;
 852                         break;
 853                 }
 854
 855                 VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
 856                         dev->vid, vq->last_avail_idx,
 857                         vq->last_avail_idx + num_buffers);
 858
 859                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
 860                                                 buf_vec, nr_vec,
 861                                                 num_buffers) < 0) {
 862                         vq->shadow_used_idx -= num_buffers;
 863                         break;
 864                 }
 865
 866                 vq->last_avail_idx += nr_descs;
 867                 if (vq->last_avail_idx >= vq->size) {
 868                         vq->last_avail_idx -= vq->size;
 869                         vq->avail_wrap_counter ^= 1;
 870                 }
 871         }
 872
 873         do_data_copy_enqueue(dev, vq);
 874
 875         if (likely(vq->shadow_used_idx)) {
 876                 flush_shadow_used_ring_packed(dev, vq);
 877                 vhost_vring_call_packed(dev, vq);
 878         }
 879
 880         return pkt_idx;
 881 }
 882
 883 static __rte_always_inline uint32_t
 884 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
 885         struct rte_mbuf **pkts, uint32_t count)
 886 {
 887         struct vhost_virtqueue *vq;
 888         uint32_t nb_tx = 0;
 889
 890         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
 891         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
 892                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
 893                         dev->vid, __func__, queue_id);
 894                 return 0;
 895         }
 896
 897         vq = dev->virtqueue[queue_id];
 898
 899         rte_spinlock_lock(&vq->access_lock);
 900
 901         if (unlikely(vq->enabled == 0))
 902                 goto out_access_unlock;
 903
 904         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 905                 vhost_user_iotlb_rd_lock(vq);
 906
 907         if (unlikely(vq->access_ok == 0))
 908                 if (unlikely(vring_translate(dev, vq) < 0))
 909                         goto out;
 910
 911         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
 912         if (count == 0)
 913                 goto out;
 914
 915         if (vq_is_packed(dev))
 916                 nb_tx = virtio_dev_rx_packed(dev, vq, pkts, count);
 917         else
 918                 nb_tx = virtio_dev_rx_split(dev, vq, pkts, count);
 919
 920 out:
 921         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 922                 vhost_user_iotlb_rd_unlock(vq);
 923
 924 out_access_unlock:
 925         rte_spinlock_unlock(&vq->access_lock);
 926
 927         return nb_tx;
 928 }
 929
 930 uint16_t
 931 rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
 932         struct rte_mbuf **pkts, uint16_t count)
 933 {
 934         struct virtio_net *dev = get_device(vid);
 935
 936         if (!dev)
 937                 return 0;
 938
 939         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
 940                 RTE_LOG(ERR, VHOST_DATA,
 941                         "(%d) %s: built-in vhost net backend is disabled.\n",
 942                         dev->vid, __func__);
 943                 return 0;
 944         }
 945
 946         return virtio_dev_rx(dev, queue_id, pkts, count);
 947 }
 948
 949 static inline bool
 950 virtio_net_with_host_offload(struct virtio_net *dev)
 951 {
 952         if (dev->features &
 953                         ((1ULL << VIRTIO_NET_F_CSUM) |
 954                          (1ULL << VIRTIO_NET_F_HOST_ECN) |
 955                          (1ULL << VIRTIO_NET_F_HOST_TSO4) |
 956                          (1ULL << VIRTIO_NET_F_HOST_TSO6) |
 957                          (1ULL << VIRTIO_NET_F_HOST_UFO)))
 958                 return true;
 959
 960         return false;
 961 }
 962
 963 static void
 964 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
 965 {
 966         struct rte_ipv4_hdr *ipv4_hdr;
 967         struct rte_ipv6_hdr *ipv6_hdr;
 968         void *l3_hdr = NULL;
 969         struct rte_ether_hdr *eth_hdr;
 970         uint16_t ethertype;
 971
 972         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 973
 974         m->l2_len = sizeof(struct rte_ether_hdr);
 975         ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
 976
 977         if (ethertype == RTE_ETHER_TYPE_VLAN) {
 978                 struct rte_vlan_hdr *vlan_hdr =
 979                         (struct rte_vlan_hdr *)(eth_hdr + 1);
 980
 981                 m->l2_len += sizeof(struct rte_vlan_hdr);
 982                 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
 983         }
 984
 985         l3_hdr = (char *)eth_hdr + m->l2_len;
 986
 987         switch (ethertype) {
 988         case RTE_ETHER_TYPE_IPV4:
 989                 ipv4_hdr = l3_hdr;
 990                 *l4_proto = ipv4_hdr->next_proto_id;
 991                 m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
 992                 *l4_hdr = (char *)l3_hdr + m->l3_len;
 993                 m->ol_flags |= PKT_TX_IPV4;
 994                 break;
 995         case RTE_ETHER_TYPE_IPV6:
 996                 ipv6_hdr = l3_hdr;
 997                 *l4_proto = ipv6_hdr->proto;
 998                 m->l3_len = sizeof(struct rte_ipv6_hdr);
 999                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1000                 m->ol_flags |= PKT_TX_IPV6;
1001                 break;
1002         default:
1003                 m->l3_len = 0;
1004                 *l4_proto = 0;
1005                 *l4_hdr = NULL;
1006                 break;
1007         }
1008 }
1009
1010 static __rte_always_inline void
1011 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
1012 {
1013         uint16_t l4_proto = 0;
1014         void *l4_hdr = NULL;
1015         struct rte_tcp_hdr *tcp_hdr = NULL;
1016
1017         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
1018                 return;
1019
1020         parse_ethernet(m, &l4_proto, &l4_hdr);
1021         if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1022                 if (hdr->csum_start == (m->l2_len + m->l3_len)) {
1023                         switch (hdr->csum_offset) {
1024                         case (offsetof(struct rte_tcp_hdr, cksum)):
1025                                 if (l4_proto == IPPROTO_TCP)
1026                                         m->ol_flags |= PKT_TX_TCP_CKSUM;
1027                                 break;
1028                         case (offsetof(struct rte_udp_hdr, dgram_cksum)):
1029                                 if (l4_proto == IPPROTO_UDP)
1030                                         m->ol_flags |= PKT_TX_UDP_CKSUM;
1031                                 break;
1032                         case (offsetof(struct rte_sctp_hdr, cksum)):
1033                                 if (l4_proto == IPPROTO_SCTP)
1034                                         m->ol_flags |= PKT_TX_SCTP_CKSUM;
1035                                 break;
1036                         default:
1037                                 break;
1038                         }
1039                 }
1040         }
1041
1042         if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1043                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1044                 case VIRTIO_NET_HDR_GSO_TCPV4:
1045                 case VIRTIO_NET_HDR_GSO_TCPV6:
1046                         tcp_hdr = l4_hdr;
1047                         m->ol_flags |= PKT_TX_TCP_SEG;
1048                         m->tso_segsz = hdr->gso_size;
1049                         m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
1050                         break;
1051                 case VIRTIO_NET_HDR_GSO_UDP:
1052                         m->ol_flags |= PKT_TX_UDP_SEG;
1053                         m->tso_segsz = hdr->gso_size;
1054                         m->l4_len = sizeof(struct rte_udp_hdr);
1055                         break;
1056                 default:
1057                         RTE_LOG(WARNING, VHOST_DATA,
1058                                 "unsupported gso type %u.\n", hdr->gso_type);
1059                         break;
1060                 }
1061         }
1062 }
1063
1064 static __rte_noinline void
1065 copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr,
1066                 struct buf_vector *buf_vec)
1067 {
1068         uint64_t len;
1069         uint64_t remain = sizeof(struct virtio_net_hdr);
1070         uint64_t src;
1071         uint64_t dst = (uint64_t)(uintptr_t)hdr;
1072
1073         while (remain) {
1074                 len = RTE_MIN(remain, buf_vec->buf_len);
1075                 src = buf_vec->buf_addr;
1076                 rte_memcpy((void *)(uintptr_t)dst,
1077                                 (void *)(uintptr_t)src, len);
1078
1079                 remain -= len;
1080                 dst += len;
1081                 buf_vec++;
1082         }
1083 }
1084
1085 static __rte_always_inline int
1086 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
1087                   struct buf_vector *buf_vec, uint16_t nr_vec,
1088                   struct rte_mbuf *m, struct rte_mempool *mbuf_pool)
1089 {
1090         uint32_t buf_avail, buf_offset;
1091         uint64_t buf_addr, buf_iova, buf_len;
1092         uint32_t mbuf_avail, mbuf_offset;
1093         uint32_t cpy_len;
1094         struct rte_mbuf *cur = m, *prev = m;
1095         struct virtio_net_hdr tmp_hdr;
1096         struct virtio_net_hdr *hdr = NULL;
1097         /* A counter to avoid desc dead loop chain */
1098         uint16_t vec_idx = 0;
1099         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
1100         int error = 0;
1101
1102         buf_addr = buf_vec[vec_idx].buf_addr;
1103         buf_iova = buf_vec[vec_idx].buf_iova;
1104         buf_len = buf_vec[vec_idx].buf_len;
1105
1106         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
1107                 error = -1;
1108                 goto out;
1109         }
1110
1111         if (virtio_net_with_host_offload(dev)) {
1112                 if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
1113                         /*
1114                          * No luck, the virtio-net header doesn't fit
1115                          * in a contiguous virtual area.
1116                          */
1117                         copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec);
1118                         hdr = &tmp_hdr;
1119                 } else {
1120                         hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);
1121                 }
1122         }
1123
1124         /*
1125          * A virtio driver normally uses at least 2 desc buffers
1126          * for Tx: the first for storing the header, and others
1127          * for storing the data.
1128          */
1129         if (unlikely(buf_len < dev->vhost_hlen)) {
1130                 buf_offset = dev->vhost_hlen - buf_len;
1131                 vec_idx++;
1132                 buf_addr = buf_vec[vec_idx].buf_addr;
1133                 buf_iova = buf_vec[vec_idx].buf_iova;
1134                 buf_len = buf_vec[vec_idx].buf_len;
1135                 buf_avail  = buf_len - buf_offset;
1136         } else if (buf_len == dev->vhost_hlen) {
1137                 if (unlikely(++vec_idx >= nr_vec))
1138                         goto out;
1139                 buf_addr = buf_vec[vec_idx].buf_addr;
1140                 buf_iova = buf_vec[vec_idx].buf_iova;
1141                 buf_len = buf_vec[vec_idx].buf_len;
1142
1143                 buf_offset = 0;
1144                 buf_avail = buf_len;
1145         } else {
1146                 buf_offset = dev->vhost_hlen;
1147                 buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
1148         }
1149
1150         PRINT_PACKET(dev,
1151                         (uintptr_t)(buf_addr + buf_offset),
1152                         (uint32_t)buf_avail, 0);
1153
1154         mbuf_offset = 0;
1155         mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
1156         while (1) {
1157                 uint64_t hpa;
1158
1159                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
1160
1161                 /*
1162                  * A desc buf might across two host physical pages that are
1163                  * not continuous. In such case (gpa_to_hpa returns 0), data
1164                  * will be copied even though zero copy is enabled.
1165                  */
1166                 if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,
1167                                         buf_iova + buf_offset, cpy_len)))) {
1168                         cur->data_len = cpy_len;
1169                         cur->data_off = 0;
1170                         cur->buf_addr =
1171                                 (void *)(uintptr_t)(buf_addr + buf_offset);
1172                         cur->buf_iova = hpa;
1173
1174                         /*
1175                          * In zero copy mode, one mbuf can only reference data
1176                          * for one or partial of one desc buff.
1177                          */
1178                         mbuf_avail = cpy_len;
1179                 } else {
1180                         if (likely(cpy_len > MAX_BATCH_LEN ||
1181                                    vq->batch_copy_nb_elems >= vq->size ||
1182                                    (hdr && cur == m))) {
1183                                 rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
1184                                                                    mbuf_offset),
1185                                            (void *)((uintptr_t)(buf_addr +
1186                                                            buf_offset)),
1187                                            cpy_len);
1188                         } else {
1189                                 batch_copy[vq->batch_copy_nb_elems].dst =
1190                                         rte_pktmbuf_mtod_offset(cur, void *,
1191                                                                 mbuf_offset);
1192                                 batch_copy[vq->batch_copy_nb_elems].src =
1193                                         (void *)((uintptr_t)(buf_addr +
1194                                                                 buf_offset));
1195                                 batch_copy[vq->batch_copy_nb_elems].len =
1196                                         cpy_len;
1197                                 vq->batch_copy_nb_elems++;
1198                         }
1199                 }
1200
1201                 mbuf_avail  -= cpy_len;
1202                 mbuf_offset += cpy_len;
1203                 buf_avail -= cpy_len;
1204                 buf_offset += cpy_len;
1205
1206                 /* This buf reaches to its end, get the next one */
1207                 if (buf_avail == 0) {
1208                         if (++vec_idx >= nr_vec)
1209                                 break;
1210
1211                         buf_addr = buf_vec[vec_idx].buf_addr;
1212                         buf_iova = buf_vec[vec_idx].buf_iova;
1213                         buf_len = buf_vec[vec_idx].buf_len;
1214
1215                         buf_offset = 0;
1216                         buf_avail  = buf_len;
1217
1218                         PRINT_PACKET(dev, (uintptr_t)buf_addr,
1219                                         (uint32_t)buf_avail, 0);
1220                 }
1221
1222                 /*
1223                  * This mbuf reaches to its end, get a new one
1224                  * to hold more data.
1225                  */
1226                 if (mbuf_avail == 0) {
1227                         cur = rte_pktmbuf_alloc(mbuf_pool);
1228                         if (unlikely(cur == NULL)) {
1229                                 RTE_LOG(ERR, VHOST_DATA, "Failed to "
1230                                         "allocate memory for mbuf.\n");
1231                                 error = -1;
1232                                 goto out;
1233                         }
1234                         if (unlikely(dev->dequeue_zero_copy))
1235                                 rte_mbuf_refcnt_update(cur, 1);
1236
1237                         prev->next = cur;
1238                         prev->data_len = mbuf_offset;
1239                         m->nb_segs += 1;
1240                         m->pkt_len += mbuf_offset;
1241                         prev = cur;
1242
1243                         mbuf_offset = 0;
1244                         mbuf_avail  = cur->buf_len - RTE_PKTMBUF_HEADROOM;
1245                 }
1246         }
1247
1248         prev->data_len = mbuf_offset;
1249         m->pkt_len    += mbuf_offset;
1250
1251         if (hdr)
1252                 vhost_dequeue_offload(hdr, m);
1253
1254 out:
1255
1256         return error;
1257 }
1258
1259 static __rte_always_inline struct zcopy_mbuf *
1260 get_zmbuf(struct vhost_virtqueue *vq)
1261 {
1262         uint16_t i;
1263         uint16_t last;
1264         int tries = 0;
1265
1266         /* search [last_zmbuf_idx, zmbuf_size) */
1267         i = vq->last_zmbuf_idx;
1268         last = vq->zmbuf_size;
1269
1270 again:
1271         for (; i < last; i++) {
1272                 if (vq->zmbufs[i].in_use == 0) {
1273                         vq->last_zmbuf_idx = i + 1;
1274                         vq->zmbufs[i].in_use = 1;
1275                         return &vq->zmbufs[i];
1276                 }
1277         }
1278
1279         tries++;
1280         if (tries == 1) {
1281                 /* search [0, last_zmbuf_idx) */
1282                 i = 0;
1283                 last = vq->last_zmbuf_idx;
1284                 goto again;
1285         }
1286
1287         return NULL;
1288 }
1289
1290 static __rte_noinline uint16_t
1291 virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
1292         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1293 {
1294         uint16_t i;
1295         uint16_t free_entries;
1296
1297         if (unlikely(dev->dequeue_zero_copy)) {
1298                 struct zcopy_mbuf *zmbuf, *next;
1299
1300                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1301                      zmbuf != NULL; zmbuf = next) {
1302                         next = TAILQ_NEXT(zmbuf, next);
1303
1304                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1305                                 update_shadow_used_ring_split(vq,
1306                                                 zmbuf->desc_idx, 0);
1307                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1308                                 restore_mbuf(zmbuf->mbuf);
1309                                 rte_pktmbuf_free(zmbuf->mbuf);
1310                                 put_zmbuf(zmbuf);
1311                                 vq->nr_zmbuf -= 1;
1312                         }
1313                 }
1314
1315                 if (likely(vq->shadow_used_idx)) {
1316                         flush_shadow_used_ring_split(dev, vq);
1317                         vhost_vring_call_split(dev, vq);
1318                 }
1319         }
1320
1321         free_entries = *((volatile uint16_t *)&vq->avail->idx) -
1322                         vq->last_avail_idx;
1323         if (free_entries == 0)
1324                 return 0;
1325
1326         /*
1327          * The ordering between avail index and
1328          * desc reads needs to be enforced.
1329          */
1330         rte_smp_rmb();
1331
1332         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
1333
1334         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1335
1336         count = RTE_MIN(count, MAX_PKT_BURST);
1337         count = RTE_MIN(count, free_entries);
1338         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
1339                         dev->vid, count);
1340
1341         for (i = 0; i < count; i++) {
1342                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
1343                 uint16_t head_idx;
1344                 uint32_t dummy_len;
1345                 uint16_t nr_vec = 0;
1346                 int err;
1347
1348                 if (unlikely(fill_vec_buf_split(dev, vq,
1349                                                 vq->last_avail_idx + i,
1350                                                 &nr_vec, buf_vec,
1351                                                 &head_idx, &dummy_len,
1352                                                 VHOST_ACCESS_RO) < 0))
1353                         break;
1354
1355                 if (likely(dev->dequeue_zero_copy == 0))
1356                         update_shadow_used_ring_split(vq, head_idx, 0);
1357
1358                 pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
1359                 if (unlikely(pkts[i] == NULL)) {
1360                         RTE_LOG(ERR, VHOST_DATA,
1361                                 "Failed to allocate memory for mbuf.\n");
1362                         break;
1363                 }
1364
1365                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
1366                                 mbuf_pool);
1367                 if (unlikely(err)) {
1368                         rte_pktmbuf_free(pkts[i]);
1369                         break;
1370                 }
1371
1372                 if (unlikely(dev->dequeue_zero_copy)) {
1373                         struct zcopy_mbuf *zmbuf;
1374
1375                         zmbuf = get_zmbuf(vq);
1376                         if (!zmbuf) {
1377                                 rte_pktmbuf_free(pkts[i]);
1378                                 break;
1379                         }
1380                         zmbuf->mbuf = pkts[i];
1381                         zmbuf->desc_idx = head_idx;
1382
1383                         /*
1384                          * Pin lock the mbuf; we will check later to see
1385                          * whether the mbuf is freed (when we are the last
1386                          * user) or not. If that's the case, we then could
1387                          * update the used ring safely.
1388                          */
1389                         rte_mbuf_refcnt_update(pkts[i], 1);
1390
1391                         vq->nr_zmbuf += 1;
1392                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1393                 }
1394         }
1395         vq->last_avail_idx += i;
1396
1397         if (likely(dev->dequeue_zero_copy == 0)) {
1398                 do_data_copy_dequeue(vq);
1399                 if (unlikely(i < count))
1400                         vq->shadow_used_idx = i;
1401                 if (likely(vq->shadow_used_idx)) {
1402                         flush_shadow_used_ring_split(dev, vq);
1403                         vhost_vring_call_split(dev, vq);
1404                 }
1405         }
1406
1407         return i;
1408 }
1409
1410 static __rte_noinline uint16_t
1411 virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
1412         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1413 {
1414         uint16_t i;
1415
1416         if (unlikely(dev->dequeue_zero_copy)) {
1417                 struct zcopy_mbuf *zmbuf, *next;
1418
1419                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1420                      zmbuf != NULL; zmbuf = next) {
1421                         next = TAILQ_NEXT(zmbuf, next);
1422
1423                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1424                                 update_shadow_used_ring_packed(vq,
1425                                                 zmbuf->desc_idx,
1426                                                 0,
1427                                                 zmbuf->desc_count);
1428
1429                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1430                                 restore_mbuf(zmbuf->mbuf);
1431                                 rte_pktmbuf_free(zmbuf->mbuf);
1432                                 put_zmbuf(zmbuf);
1433                                 vq->nr_zmbuf -= 1;
1434                         }
1435                 }
1436
1437                 if (likely(vq->shadow_used_idx)) {
1438                         flush_shadow_used_ring_packed(dev, vq);
1439                         vhost_vring_call_packed(dev, vq);
1440                 }
1441         }
1442
1443         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1444
1445         count = RTE_MIN(count, MAX_PKT_BURST);
1446         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
1447                         dev->vid, count);
1448
1449         for (i = 0; i < count; i++) {
1450                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
1451                 uint16_t buf_id;
1452                 uint32_t dummy_len;
1453                 uint16_t desc_count, nr_vec = 0;
1454                 int err;
1455
1456                 if (unlikely(fill_vec_buf_packed(dev, vq,
1457                                                 vq->last_avail_idx, &desc_count,
1458                                                 buf_vec, &nr_vec,
1459                                                 &buf_id, &dummy_len,
1460                                                 VHOST_ACCESS_RO) < 0))
1461                         break;
1462
1463                 if (likely(dev->dequeue_zero_copy == 0))
1464                         update_shadow_used_ring_packed(vq, buf_id, 0,
1465                                         desc_count);
1466
1467                 pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
1468                 if (unlikely(pkts[i] == NULL)) {
1469                         RTE_LOG(ERR, VHOST_DATA,
1470                                 "Failed to allocate memory for mbuf.\n");
1471                         break;
1472                 }
1473
1474                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
1475                                 mbuf_pool);
1476                 if (unlikely(err)) {
1477                         rte_pktmbuf_free(pkts[i]);
1478                         break;
1479                 }
1480
1481                 if (unlikely(dev->dequeue_zero_copy)) {
1482                         struct zcopy_mbuf *zmbuf;
1483
1484                         zmbuf = get_zmbuf(vq);
1485                         if (!zmbuf) {
1486                                 rte_pktmbuf_free(pkts[i]);
1487                                 break;
1488                         }
1489                         zmbuf->mbuf = pkts[i];
1490                         zmbuf->desc_idx = buf_id;
1491                         zmbuf->desc_count = desc_count;
1492
1493                         /*
1494                          * Pin lock the mbuf; we will check later to see
1495                          * whether the mbuf is freed (when we are the last
1496                          * user) or not. If that's the case, we then could
1497                          * update the used ring safely.
1498                          */
1499                         rte_mbuf_refcnt_update(pkts[i], 1);
1500
1501                         vq->nr_zmbuf += 1;
1502                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1503                 }
1504
1505                 vq->last_avail_idx += desc_count;
1506                 if (vq->last_avail_idx >= vq->size) {
1507                         vq->last_avail_idx -= vq->size;
1508                         vq->avail_wrap_counter ^= 1;
1509                 }
1510         }
1511
1512         if (likely(dev->dequeue_zero_copy == 0)) {
1513                 do_data_copy_dequeue(vq);
1514                 if (unlikely(i < count))
1515                         vq->shadow_used_idx = i;
1516                 if (likely(vq->shadow_used_idx)) {
1517                         flush_shadow_used_ring_packed(dev, vq);
1518                         vhost_vring_call_packed(dev, vq);
1519                 }
1520         }
1521
1522         return i;
1523 }
1524
1525 uint16_t
1526 rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
1527         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1528 {
1529         struct virtio_net *dev;
1530         struct rte_mbuf *rarp_mbuf = NULL;
1531         struct vhost_virtqueue *vq;
1532
1533         dev = get_device(vid);
1534         if (!dev)
1535                 return 0;
1536
1537         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
1538                 RTE_LOG(ERR, VHOST_DATA,
1539                         "(%d) %s: built-in vhost net backend is disabled.\n",
1540                         dev->vid, __func__);
1541                 return 0;
1542         }
1543
1544         if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
1545                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
1546                         dev->vid, __func__, queue_id);
1547                 return 0;
1548         }
1549
1550         vq = dev->virtqueue[queue_id];
1551
1552         if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
1553                 return 0;
1554
1555         if (unlikely(vq->enabled == 0)) {
1556                 count = 0;
1557                 goto out_access_unlock;
1558         }
1559
1560         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1561                 vhost_user_iotlb_rd_lock(vq);
1562
1563         if (unlikely(vq->access_ok == 0))
1564                 if (unlikely(vring_translate(dev, vq) < 0)) {
1565                         count = 0;
1566                         goto out;
1567                 }
1568
1569         /*
1570          * Construct a RARP broadcast packet, and inject it to the "pkts"
1571          * array, to looks like that guest actually send such packet.
1572          *
1573          * Check user_send_rarp() for more information.
1574          *
1575          * broadcast_rarp shares a cacheline in the virtio_net structure
1576          * with some fields that are accessed during enqueue and
1577          * rte_atomic16_cmpset() causes a write if using cmpxchg. This could
1578          * result in false sharing between enqueue and dequeue.
1579          *
1580          * Prevent unnecessary false sharing by reading broadcast_rarp first
1581          * and only performing cmpset if the read indicates it is likely to
1582          * be set.
1583          */
1584         if (unlikely(rte_atomic16_read(&dev->broadcast_rarp) &&
1585                         rte_atomic16_cmpset((volatile uint16_t *)
1586                                 &dev->broadcast_rarp.cnt, 1, 0))) {
1587
1588                 rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
1589                 if (rarp_mbuf == NULL) {
1590                         RTE_LOG(ERR, VHOST_DATA,
1591                                 "Failed to make RARP packet.\n");
1592                         count = 0;
1593                         goto out;
1594                 }
1595                 count -= 1;
1596         }
1597
1598         if (vq_is_packed(dev))
1599                 count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count);
1600         else
1601                 count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);
1602
1603 out:
1604         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1605                 vhost_user_iotlb_rd_unlock(vq);
1606
1607 out_access_unlock:
1608         rte_spinlock_unlock(&vq->access_lock);
1609
1610         if (unlikely(rarp_mbuf != NULL)) {
1611                 /*
1612                  * Inject it to the head of "pkts" array, so that switch's mac
1613                  * learning table will get updated first.
1614                  */
1615                 memmove(&pkts[1], pkts, count * sizeof(struct rte_mbuf *));
1616                 pkts[0] = rarp_mbuf;
1617                 count += 1;
1618         }
1619
1620         return count;
1621 }