lib/vhost/virtio_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdbool.h>
   7 #include <linux/virtio_net.h>
   8
   9 #include <rte_mbuf.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_ether.h>
  12 #include <rte_ip.h>
  13 #include <rte_vhost.h>
  14 #include <rte_tcp.h>
  15 #include <rte_udp.h>
  16 #include <rte_sctp.h>
  17 #include <rte_arp.h>
  18 #include <rte_spinlock.h>
  19 #include <rte_malloc.h>
  20 #include <rte_vhost_async.h>
  21
  22 #include "iotlb.h"
  23 #include "vhost.h"
  24
  25 #define MAX_BATCH_LEN 256
  26
  27 #define VHOST_ASYNC_BATCH_THRESHOLD 32
  28
  29 static  __rte_always_inline bool
  30 rxvq_is_mergeable(struct virtio_net *dev)
  31 {
  32         return dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF);
  33 }
  34
  35 static  __rte_always_inline bool
  36 virtio_net_is_inorder(struct virtio_net *dev)
  37 {
  38         return dev->features & (1ULL << VIRTIO_F_IN_ORDER);
  39 }
  40
  41 static bool
  42 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
  43 {
  44         return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring;
  45 }
  46
  47 static inline void
  48 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
  49 {
  50         struct batch_copy_elem *elem = vq->batch_copy_elems;
  51         uint16_t count = vq->batch_copy_nb_elems;
  52         int i;
  53
  54         for (i = 0; i < count; i++) {
  55                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
  56                 vhost_log_cache_write_iova(dev, vq, elem[i].log_addr,
  57                                            elem[i].len);
  58                 PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
  59         }
  60
  61         vq->batch_copy_nb_elems = 0;
  62 }
  63
  64 static inline void
  65 do_data_copy_dequeue(struct vhost_virtqueue *vq)
  66 {
  67         struct batch_copy_elem *elem = vq->batch_copy_elems;
  68         uint16_t count = vq->batch_copy_nb_elems;
  69         int i;
  70
  71         for (i = 0; i < count; i++)
  72                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
  73
  74         vq->batch_copy_nb_elems = 0;
  75 }
  76
  77 static __rte_always_inline void
  78 do_flush_shadow_used_ring_split(struct virtio_net *dev,
  79                         struct vhost_virtqueue *vq,
  80                         uint16_t to, uint16_t from, uint16_t size)
  81 {
  82         rte_memcpy(&vq->used->ring[to],
  83                         &vq->shadow_used_split[from],
  84                         size * sizeof(struct vring_used_elem));
  85         vhost_log_cache_used_vring(dev, vq,
  86                         offsetof(struct vring_used, ring[to]),
  87                         size * sizeof(struct vring_used_elem));
  88 }
  89
  90 static __rte_always_inline void
  91 flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
  92 {
  93         uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
  94
  95         if (used_idx + vq->shadow_used_idx <= vq->size) {
  96                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0,
  97                                           vq->shadow_used_idx);
  98         } else {
  99                 uint16_t size;
 100
 101                 /* update used ring interval [used_idx, vq->size] */
 102                 size = vq->size - used_idx;
 103                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size);
 104
 105                 /* update the left half used ring interval [0, left_size] */
 106                 do_flush_shadow_used_ring_split(dev, vq, 0, size,
 107                                           vq->shadow_used_idx - size);
 108         }
 109         vq->last_used_idx += vq->shadow_used_idx;
 110
 111         vhost_log_cache_sync(dev, vq);
 112
 113         __atomic_add_fetch(&vq->used->idx, vq->shadow_used_idx,
 114                            __ATOMIC_RELEASE);
 115         vq->shadow_used_idx = 0;
 116         vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
 117                 sizeof(vq->used->idx));
 118 }
 119
 120 static __rte_always_inline void
 121 update_shadow_used_ring_split(struct vhost_virtqueue *vq,
 122                          uint16_t desc_idx, uint32_t len)
 123 {
 124         uint16_t i = vq->shadow_used_idx++;
 125
 126         vq->shadow_used_split[i].id  = desc_idx;
 127         vq->shadow_used_split[i].len = len;
 128 }
 129
 130 static __rte_always_inline void
 131 vhost_flush_enqueue_shadow_packed(struct virtio_net *dev,
 132                                   struct vhost_virtqueue *vq)
 133 {
 134         int i;
 135         uint16_t used_idx = vq->last_used_idx;
 136         uint16_t head_idx = vq->last_used_idx;
 137         uint16_t head_flags = 0;
 138
 139         /* Split loop in two to save memory barriers */
 140         for (i = 0; i < vq->shadow_used_idx; i++) {
 141                 vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id;
 142                 vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len;
 143
 144                 used_idx += vq->shadow_used_packed[i].count;
 145                 if (used_idx >= vq->size)
 146                         used_idx -= vq->size;
 147         }
 148
 149         /* The ordering for storing desc flags needs to be enforced. */
 150         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 151
 152         for (i = 0; i < vq->shadow_used_idx; i++) {
 153                 uint16_t flags;
 154
 155                 if (vq->shadow_used_packed[i].len)
 156                         flags = VRING_DESC_F_WRITE;
 157                 else
 158                         flags = 0;
 159
 160                 if (vq->used_wrap_counter) {
 161                         flags |= VRING_DESC_F_USED;
 162                         flags |= VRING_DESC_F_AVAIL;
 163                 } else {
 164                         flags &= ~VRING_DESC_F_USED;
 165                         flags &= ~VRING_DESC_F_AVAIL;
 166                 }
 167
 168                 if (i > 0) {
 169                         vq->desc_packed[vq->last_used_idx].flags = flags;
 170
 171                         vhost_log_cache_used_vring(dev, vq,
 172                                         vq->last_used_idx *
 173                                         sizeof(struct vring_packed_desc),
 174                                         sizeof(struct vring_packed_desc));
 175                 } else {
 176                         head_idx = vq->last_used_idx;
 177                         head_flags = flags;
 178                 }
 179
 180                 vq_inc_last_used_packed(vq, vq->shadow_used_packed[i].count);
 181         }
 182
 183         vq->desc_packed[head_idx].flags = head_flags;
 184
 185         vhost_log_cache_used_vring(dev, vq,
 186                                 head_idx *
 187                                 sizeof(struct vring_packed_desc),
 188                                 sizeof(struct vring_packed_desc));
 189
 190         vq->shadow_used_idx = 0;
 191         vhost_log_cache_sync(dev, vq);
 192 }
 193
 194 static __rte_always_inline void
 195 vhost_flush_dequeue_shadow_packed(struct virtio_net *dev,
 196                                   struct vhost_virtqueue *vq)
 197 {
 198         struct vring_used_elem_packed *used_elem = &vq->shadow_used_packed[0];
 199
 200         vq->desc_packed[vq->shadow_last_used_idx].id = used_elem->id;
 201         /* desc flags is the synchronization point for virtio packed vring */
 202         __atomic_store_n(&vq->desc_packed[vq->shadow_last_used_idx].flags,
 203                          used_elem->flags, __ATOMIC_RELEASE);
 204
 205         vhost_log_cache_used_vring(dev, vq, vq->shadow_last_used_idx *
 206                                    sizeof(struct vring_packed_desc),
 207                                    sizeof(struct vring_packed_desc));
 208         vq->shadow_used_idx = 0;
 209         vhost_log_cache_sync(dev, vq);
 210 }
 211
 212 static __rte_always_inline void
 213 vhost_flush_enqueue_batch_packed(struct virtio_net *dev,
 214                                  struct vhost_virtqueue *vq,
 215                                  uint64_t *lens,
 216                                  uint16_t *ids)
 217 {
 218         uint16_t i;
 219         uint16_t flags;
 220         uint16_t last_used_idx = vq->last_used_idx;
 221         struct vring_packed_desc *desc_base = &vq->desc_packed[last_used_idx];
 222
 223         if (vq->shadow_used_idx) {
 224                 do_data_copy_enqueue(dev, vq);
 225                 vhost_flush_enqueue_shadow_packed(dev, vq);
 226         }
 227
 228         flags = PACKED_DESC_ENQUEUE_USED_FLAG(vq->used_wrap_counter);
 229
 230         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
 231                 desc_base[i].id = ids[i];
 232                 desc_base[i].len = lens[i];
 233         }
 234
 235         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 236
 237         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
 238                 desc_base[i].flags = flags;
 239         }
 240
 241         vhost_log_cache_used_vring(dev, vq, last_used_idx *
 242                                    sizeof(struct vring_packed_desc),
 243                                    sizeof(struct vring_packed_desc) *
 244                                    PACKED_BATCH_SIZE);
 245         vhost_log_cache_sync(dev, vq);
 246
 247         vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);
 248 }
 249
 250 static __rte_always_inline void
 251 vhost_shadow_dequeue_batch_packed_inorder(struct vhost_virtqueue *vq,
 252                                           uint16_t id)
 253 {
 254         vq->shadow_used_packed[0].id = id;
 255
 256         if (!vq->shadow_used_idx) {
 257                 vq->shadow_last_used_idx = vq->last_used_idx;
 258                 vq->shadow_used_packed[0].flags =
 259                         PACKED_DESC_DEQUEUE_USED_FLAG(vq->used_wrap_counter);
 260                 vq->shadow_used_packed[0].len = 0;
 261                 vq->shadow_used_packed[0].count = 1;
 262                 vq->shadow_used_idx++;
 263         }
 264
 265         vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);
 266 }
 267
 268 static __rte_always_inline void
 269 vhost_shadow_dequeue_batch_packed(struct virtio_net *dev,
 270                                   struct vhost_virtqueue *vq,
 271                                   uint16_t *ids)
 272 {
 273         uint16_t flags;
 274         uint16_t i;
 275         uint16_t begin;
 276
 277         flags = PACKED_DESC_DEQUEUE_USED_FLAG(vq->used_wrap_counter);
 278
 279         if (!vq->shadow_used_idx) {
 280                 vq->shadow_last_used_idx = vq->last_used_idx;
 281                 vq->shadow_used_packed[0].id  = ids[0];
 282                 vq->shadow_used_packed[0].len = 0;
 283                 vq->shadow_used_packed[0].count = 1;
 284                 vq->shadow_used_packed[0].flags = flags;
 285                 vq->shadow_used_idx++;
 286                 begin = 1;
 287         } else
 288                 begin = 0;
 289
 290         vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE) {
 291                 vq->desc_packed[vq->last_used_idx + i].id = ids[i];
 292                 vq->desc_packed[vq->last_used_idx + i].len = 0;
 293         }
 294
 295         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 296         vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE)
 297                 vq->desc_packed[vq->last_used_idx + i].flags = flags;
 298
 299         vhost_log_cache_used_vring(dev, vq, vq->last_used_idx *
 300                                    sizeof(struct vring_packed_desc),
 301                                    sizeof(struct vring_packed_desc) *
 302                                    PACKED_BATCH_SIZE);
 303         vhost_log_cache_sync(dev, vq);
 304
 305         vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);
 306 }
 307
 308 static __rte_always_inline void
 309 vhost_shadow_dequeue_single_packed(struct vhost_virtqueue *vq,
 310                                    uint16_t buf_id,
 311                                    uint16_t count)
 312 {
 313         uint16_t flags;
 314
 315         flags = vq->desc_packed[vq->last_used_idx].flags;
 316         if (vq->used_wrap_counter) {
 317                 flags |= VRING_DESC_F_USED;
 318                 flags |= VRING_DESC_F_AVAIL;
 319         } else {
 320                 flags &= ~VRING_DESC_F_USED;
 321                 flags &= ~VRING_DESC_F_AVAIL;
 322         }
 323
 324         if (!vq->shadow_used_idx) {
 325                 vq->shadow_last_used_idx = vq->last_used_idx;
 326
 327                 vq->shadow_used_packed[0].id  = buf_id;
 328                 vq->shadow_used_packed[0].len = 0;
 329                 vq->shadow_used_packed[0].flags = flags;
 330                 vq->shadow_used_idx++;
 331         } else {
 332                 vq->desc_packed[vq->last_used_idx].id = buf_id;
 333                 vq->desc_packed[vq->last_used_idx].len = 0;
 334                 vq->desc_packed[vq->last_used_idx].flags = flags;
 335         }
 336
 337         vq_inc_last_used_packed(vq, count);
 338 }
 339
 340 static __rte_always_inline void
 341 vhost_shadow_dequeue_single_packed_inorder(struct vhost_virtqueue *vq,
 342                                            uint16_t buf_id,
 343                                            uint16_t count)
 344 {
 345         uint16_t flags;
 346
 347         vq->shadow_used_packed[0].id = buf_id;
 348
 349         flags = vq->desc_packed[vq->last_used_idx].flags;
 350         if (vq->used_wrap_counter) {
 351                 flags |= VRING_DESC_F_USED;
 352                 flags |= VRING_DESC_F_AVAIL;
 353         } else {
 354                 flags &= ~VRING_DESC_F_USED;
 355                 flags &= ~VRING_DESC_F_AVAIL;
 356         }
 357
 358         if (!vq->shadow_used_idx) {
 359                 vq->shadow_last_used_idx = vq->last_used_idx;
 360                 vq->shadow_used_packed[0].len = 0;
 361                 vq->shadow_used_packed[0].flags = flags;
 362                 vq->shadow_used_idx++;
 363         }
 364
 365         vq_inc_last_used_packed(vq, count);
 366 }
 367
 368 static __rte_always_inline void
 369 vhost_shadow_enqueue_single_packed(struct virtio_net *dev,
 370                                    struct vhost_virtqueue *vq,
 371                                    uint32_t len[],
 372                                    uint16_t id[],
 373                                    uint16_t count[],
 374                                    uint16_t num_buffers)
 375 {
 376         uint16_t i;
 377         for (i = 0; i < num_buffers; i++) {
 378                 /* enqueue shadow flush action aligned with batch num */
 379                 if (!vq->shadow_used_idx)
 380                         vq->shadow_aligned_idx = vq->last_used_idx &
 381                                 PACKED_BATCH_MASK;
 382                 vq->shadow_used_packed[vq->shadow_used_idx].id  = id[i];
 383                 vq->shadow_used_packed[vq->shadow_used_idx].len = len[i];
 384                 vq->shadow_used_packed[vq->shadow_used_idx].count = count[i];
 385                 vq->shadow_aligned_idx += count[i];
 386                 vq->shadow_used_idx++;
 387         }
 388
 389         if (vq->shadow_aligned_idx >= PACKED_BATCH_SIZE) {
 390                 do_data_copy_enqueue(dev, vq);
 391                 vhost_flush_enqueue_shadow_packed(dev, vq);
 392         }
 393 }
 394
 395 /* avoid write operation when necessary, to lessen cache issues */
 396 #define ASSIGN_UNLESS_EQUAL(var, val) do {      \
 397         if ((var) != (val))                     \
 398                 (var) = (val);                  \
 399 } while (0)
 400
 401 static __rte_always_inline void
 402 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 403 {
 404         uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK;
 405
 406         if (m_buf->ol_flags & PKT_TX_TCP_SEG)
 407                 csum_l4 |= PKT_TX_TCP_CKSUM;
 408
 409         if (csum_l4) {
 410                 net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
 411                 net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len;
 412
 413                 switch (csum_l4) {
 414                 case PKT_TX_TCP_CKSUM:
 415                         net_hdr->csum_offset = (offsetof(struct rte_tcp_hdr,
 416                                                 cksum));
 417                         break;
 418                 case PKT_TX_UDP_CKSUM:
 419                         net_hdr->csum_offset = (offsetof(struct rte_udp_hdr,
 420                                                 dgram_cksum));
 421                         break;
 422                 case PKT_TX_SCTP_CKSUM:
 423                         net_hdr->csum_offset = (offsetof(struct rte_sctp_hdr,
 424                                                 cksum));
 425                         break;
 426                 }
 427         } else {
 428                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0);
 429                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0);
 430                 ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0);
 431         }
 432
 433         /* IP cksum verification cannot be bypassed, then calculate here */
 434         if (m_buf->ol_flags & PKT_TX_IP_CKSUM) {
 435                 struct rte_ipv4_hdr *ipv4_hdr;
 436
 437                 ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct rte_ipv4_hdr *,
 438                                                    m_buf->l2_len);
 439                 ipv4_hdr->hdr_checksum = 0;
 440                 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
 441         }
 442
 443         if (m_buf->ol_flags & PKT_TX_TCP_SEG) {
 444                 if (m_buf->ol_flags & PKT_TX_IPV4)
 445                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
 446                 else
 447                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
 448                 net_hdr->gso_size = m_buf->tso_segsz;
 449                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len
 450                                         + m_buf->l4_len;
 451         } else if (m_buf->ol_flags & PKT_TX_UDP_SEG) {
 452                 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
 453                 net_hdr->gso_size = m_buf->tso_segsz;
 454                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len +
 455                         m_buf->l4_len;
 456         } else {
 457                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0);
 458                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0);
 459                 ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0);
 460         }
 461 }
 462
 463 static __rte_always_inline int
 464 map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 465                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 466                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 467 {
 468         uint16_t vec_id = *vec_idx;
 469
 470         while (desc_len) {
 471                 uint64_t desc_addr;
 472                 uint64_t desc_chunck_len = desc_len;
 473
 474                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 475                         return -1;
 476
 477                 desc_addr = vhost_iova_to_vva(dev, vq,
 478                                 desc_iova,
 479                                 &desc_chunck_len,
 480                                 perm);
 481                 if (unlikely(!desc_addr))
 482                         return -1;
 483
 484                 rte_prefetch0((void *)(uintptr_t)desc_addr);
 485
 486                 buf_vec[vec_id].buf_iova = desc_iova;
 487                 buf_vec[vec_id].buf_addr = desc_addr;
 488                 buf_vec[vec_id].buf_len  = desc_chunck_len;
 489
 490                 desc_len -= desc_chunck_len;
 491                 desc_iova += desc_chunck_len;
 492                 vec_id++;
 493         }
 494         *vec_idx = vec_id;
 495
 496         return 0;
 497 }
 498
 499 static __rte_always_inline int
 500 fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 501                          uint32_t avail_idx, uint16_t *vec_idx,
 502                          struct buf_vector *buf_vec, uint16_t *desc_chain_head,
 503                          uint32_t *desc_chain_len, uint8_t perm)
 504 {
 505         uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
 506         uint16_t vec_id = *vec_idx;
 507         uint32_t len    = 0;
 508         uint64_t dlen;
 509         uint32_t nr_descs = vq->size;
 510         uint32_t cnt    = 0;
 511         struct vring_desc *descs = vq->desc;
 512         struct vring_desc *idesc = NULL;
 513
 514         if (unlikely(idx >= vq->size))
 515                 return -1;
 516
 517         *desc_chain_head = idx;
 518
 519         if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
 520                 dlen = vq->desc[idx].len;
 521                 nr_descs = dlen / sizeof(struct vring_desc);
 522                 if (unlikely(nr_descs > vq->size))
 523                         return -1;
 524
 525                 descs = (struct vring_desc *)(uintptr_t)
 526                         vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
 527                                                 &dlen,
 528                                                 VHOST_ACCESS_RO);
 529                 if (unlikely(!descs))
 530                         return -1;
 531
 532                 if (unlikely(dlen < vq->desc[idx].len)) {
 533                         /*
 534                          * The indirect desc table is not contiguous
 535                          * in process VA space, we have to copy it.
 536                          */
 537                         idesc = vhost_alloc_copy_ind_table(dev, vq,
 538                                         vq->desc[idx].addr, vq->desc[idx].len);
 539                         if (unlikely(!idesc))
 540                                 return -1;
 541
 542                         descs = idesc;
 543                 }
 544
 545                 idx = 0;
 546         }
 547
 548         while (1) {
 549                 if (unlikely(idx >= nr_descs || cnt++ >= nr_descs)) {
 550                         free_ind_table(idesc);
 551                         return -1;
 552                 }
 553
 554                 dlen = descs[idx].len;
 555                 len += dlen;
 556
 557                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 558                                                 descs[idx].addr, dlen,
 559                                                 perm))) {
 560                         free_ind_table(idesc);
 561                         return -1;
 562                 }
 563
 564                 if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
 565                         break;
 566
 567                 idx = descs[idx].next;
 568         }
 569
 570         *desc_chain_len = len;
 571         *vec_idx = vec_id;
 572
 573         if (unlikely(!!idesc))
 574                 free_ind_table(idesc);
 575
 576         return 0;
 577 }
 578
 579 /*
 580  * Returns -1 on fail, 0 on success
 581  */
 582 static inline int
 583 reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 584                                 uint32_t size, struct buf_vector *buf_vec,
 585                                 uint16_t *num_buffers, uint16_t avail_head,
 586                                 uint16_t *nr_vec)
 587 {
 588         uint16_t cur_idx;
 589         uint16_t vec_idx = 0;
 590         uint16_t max_tries, tries = 0;
 591
 592         uint16_t head_idx = 0;
 593         uint32_t len = 0;
 594
 595         *num_buffers = 0;
 596         cur_idx  = vq->last_avail_idx;
 597
 598         if (rxvq_is_mergeable(dev))
 599                 max_tries = vq->size - 1;
 600         else
 601                 max_tries = 1;
 602
 603         while (size > 0) {
 604                 if (unlikely(cur_idx == avail_head))
 605                         return -1;
 606                 /*
 607                  * if we tried all available ring items, and still
 608                  * can't get enough buf, it means something abnormal
 609                  * happened.
 610                  */
 611                 if (unlikely(++tries > max_tries))
 612                         return -1;
 613
 614                 if (unlikely(fill_vec_buf_split(dev, vq, cur_idx,
 615                                                 &vec_idx, buf_vec,
 616                                                 &head_idx, &len,
 617                                                 VHOST_ACCESS_RW) < 0))
 618                         return -1;
 619                 len = RTE_MIN(len, size);
 620                 update_shadow_used_ring_split(vq, head_idx, len);
 621                 size -= len;
 622
 623                 cur_idx++;
 624                 *num_buffers += 1;
 625         }
 626
 627         *nr_vec = vec_idx;
 628
 629         return 0;
 630 }
 631
 632 static __rte_always_inline int
 633 fill_vec_buf_packed_indirect(struct virtio_net *dev,
 634                         struct vhost_virtqueue *vq,
 635                         struct vring_packed_desc *desc, uint16_t *vec_idx,
 636                         struct buf_vector *buf_vec, uint32_t *len, uint8_t perm)
 637 {
 638         uint16_t i;
 639         uint32_t nr_descs;
 640         uint16_t vec_id = *vec_idx;
 641         uint64_t dlen;
 642         struct vring_packed_desc *descs, *idescs = NULL;
 643
 644         dlen = desc->len;
 645         descs = (struct vring_packed_desc *)(uintptr_t)
 646                 vhost_iova_to_vva(dev, vq, desc->addr, &dlen, VHOST_ACCESS_RO);
 647         if (unlikely(!descs))
 648                 return -1;
 649
 650         if (unlikely(dlen < desc->len)) {
 651                 /*
 652                  * The indirect desc table is not contiguous
 653                  * in process VA space, we have to copy it.
 654                  */
 655                 idescs = vhost_alloc_copy_ind_table(dev,
 656                                 vq, desc->addr, desc->len);
 657                 if (unlikely(!idescs))
 658                         return -1;
 659
 660                 descs = idescs;
 661         }
 662
 663         nr_descs =  desc->len / sizeof(struct vring_packed_desc);
 664         if (unlikely(nr_descs >= vq->size)) {
 665                 free_ind_table(idescs);
 666                 return -1;
 667         }
 668
 669         for (i = 0; i < nr_descs; i++) {
 670                 if (unlikely(vec_id >= BUF_VECTOR_MAX)) {
 671                         free_ind_table(idescs);
 672                         return -1;
 673                 }
 674
 675                 dlen = descs[i].len;
 676                 *len += dlen;
 677                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 678                                                 descs[i].addr, dlen,
 679                                                 perm)))
 680                         return -1;
 681         }
 682         *vec_idx = vec_id;
 683
 684         if (unlikely(!!idescs))
 685                 free_ind_table(idescs);
 686
 687         return 0;
 688 }
 689
 690 static __rte_always_inline int
 691 fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 692                                 uint16_t avail_idx, uint16_t *desc_count,
 693                                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 694                                 uint16_t *buf_id, uint32_t *len, uint8_t perm)
 695 {
 696         bool wrap_counter = vq->avail_wrap_counter;
 697         struct vring_packed_desc *descs = vq->desc_packed;
 698         uint16_t vec_id = *vec_idx;
 699         uint64_t dlen;
 700
 701         if (avail_idx < vq->last_avail_idx)
 702                 wrap_counter ^= 1;
 703
 704         /*
 705          * Perform a load-acquire barrier in desc_is_avail to
 706          * enforce the ordering between desc flags and desc
 707          * content.
 708          */
 709         if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
 710                 return -1;
 711
 712         *desc_count = 0;
 713         *len = 0;
 714
 715         while (1) {
 716                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 717                         return -1;
 718
 719                 if (unlikely(*desc_count >= vq->size))
 720                         return -1;
 721
 722                 *desc_count += 1;
 723                 *buf_id = descs[avail_idx].id;
 724
 725                 if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) {
 726                         if (unlikely(fill_vec_buf_packed_indirect(dev, vq,
 727                                                         &descs[avail_idx],
 728                                                         &vec_id, buf_vec,
 729                                                         len, perm) < 0))
 730                                 return -1;
 731                 } else {
 732                         dlen = descs[avail_idx].len;
 733                         *len += dlen;
 734
 735                         if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 736                                                         descs[avail_idx].addr,
 737                                                         dlen,
 738                                                         perm)))
 739                                 return -1;
 740                 }
 741
 742                 if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0)
 743                         break;
 744
 745                 if (++avail_idx >= vq->size) {
 746                         avail_idx -= vq->size;
 747                         wrap_counter ^= 1;
 748                 }
 749         }
 750
 751         *vec_idx = vec_id;
 752
 753         return 0;
 754 }
 755
 756 static __rte_noinline void
 757 copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 758                 struct buf_vector *buf_vec,
 759                 struct virtio_net_hdr_mrg_rxbuf *hdr)
 760 {
 761         uint64_t len;
 762         uint64_t remain = dev->vhost_hlen;
 763         uint64_t src = (uint64_t)(uintptr_t)hdr, dst;
 764         uint64_t iova = buf_vec->buf_iova;
 765
 766         while (remain) {
 767                 len = RTE_MIN(remain,
 768                                 buf_vec->buf_len);
 769                 dst = buf_vec->buf_addr;
 770                 rte_memcpy((void *)(uintptr_t)dst,
 771                                 (void *)(uintptr_t)src,
 772                                 len);
 773
 774                 PRINT_PACKET(dev, (uintptr_t)dst,
 775                                 (uint32_t)len, 0);
 776                 vhost_log_cache_write_iova(dev, vq,
 777                                 iova, len);
 778
 779                 remain -= len;
 780                 iova += len;
 781                 src += len;
 782                 buf_vec++;
 783         }
 784 }
 785
 786 static __rte_always_inline int
 787 copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 788                             struct rte_mbuf *m, struct buf_vector *buf_vec,
 789                             uint16_t nr_vec, uint16_t num_buffers)
 790 {
 791         uint32_t vec_idx = 0;
 792         uint32_t mbuf_offset, mbuf_avail;
 793         uint32_t buf_offset, buf_avail;
 794         uint64_t buf_addr, buf_iova, buf_len;
 795         uint32_t cpy_len;
 796         uint64_t hdr_addr;
 797         struct rte_mbuf *hdr_mbuf;
 798         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 799         struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
 800         int error = 0;
 801
 802         if (unlikely(m == NULL)) {
 803                 error = -1;
 804                 goto out;
 805         }
 806
 807         buf_addr = buf_vec[vec_idx].buf_addr;
 808         buf_iova = buf_vec[vec_idx].buf_iova;
 809         buf_len = buf_vec[vec_idx].buf_len;
 810
 811         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 812                 error = -1;
 813                 goto out;
 814         }
 815
 816         hdr_mbuf = m;
 817         hdr_addr = buf_addr;
 818         if (unlikely(buf_len < dev->vhost_hlen)) {
 819                 memset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf));
 820                 hdr = &tmp_hdr;
 821         } else
 822                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
 823
 824         VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n",
 825                 dev->vid, num_buffers);
 826
 827         if (unlikely(buf_len < dev->vhost_hlen)) {
 828                 buf_offset = dev->vhost_hlen - buf_len;
 829                 vec_idx++;
 830                 buf_addr = buf_vec[vec_idx].buf_addr;
 831                 buf_iova = buf_vec[vec_idx].buf_iova;
 832                 buf_len = buf_vec[vec_idx].buf_len;
 833                 buf_avail = buf_len - buf_offset;
 834         } else {
 835                 buf_offset = dev->vhost_hlen;
 836                 buf_avail = buf_len - dev->vhost_hlen;
 837         }
 838
 839         mbuf_avail  = rte_pktmbuf_data_len(m);
 840         mbuf_offset = 0;
 841         while (mbuf_avail != 0 || m->next != NULL) {
 842                 /* done with current buf, get the next one */
 843                 if (buf_avail == 0) {
 844                         vec_idx++;
 845                         if (unlikely(vec_idx >= nr_vec)) {
 846                                 error = -1;
 847                                 goto out;
 848                         }
 849
 850                         buf_addr = buf_vec[vec_idx].buf_addr;
 851                         buf_iova = buf_vec[vec_idx].buf_iova;
 852                         buf_len = buf_vec[vec_idx].buf_len;
 853
 854                         buf_offset = 0;
 855                         buf_avail  = buf_len;
 856                 }
 857
 858                 /* done with current mbuf, get the next one */
 859                 if (mbuf_avail == 0) {
 860                         m = m->next;
 861
 862                         mbuf_offset = 0;
 863                         mbuf_avail  = rte_pktmbuf_data_len(m);
 864                 }
 865
 866                 if (hdr_addr) {
 867                         virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
 868                         if (rxvq_is_mergeable(dev))
 869                                 ASSIGN_UNLESS_EQUAL(hdr->num_buffers,
 870                                                 num_buffers);
 871
 872                         if (unlikely(hdr == &tmp_hdr)) {
 873                                 copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr);
 874                         } else {
 875                                 PRINT_PACKET(dev, (uintptr_t)hdr_addr,
 876                                                 dev->vhost_hlen, 0);
 877                                 vhost_log_cache_write_iova(dev, vq,
 878                                                 buf_vec[0].buf_iova,
 879                                                 dev->vhost_hlen);
 880                         }
 881
 882                         hdr_addr = 0;
 883                 }
 884
 885                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 886
 887                 if (likely(cpy_len > MAX_BATCH_LEN ||
 888                                         vq->batch_copy_nb_elems >= vq->size)) {
 889                         rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),
 890                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 891                                 cpy_len);
 892                         vhost_log_cache_write_iova(dev, vq,
 893                                                    buf_iova + buf_offset,
 894                                                    cpy_len);
 895                         PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),
 896                                 cpy_len, 0);
 897                 } else {
 898                         batch_copy[vq->batch_copy_nb_elems].dst =
 899                                 (void *)((uintptr_t)(buf_addr + buf_offset));
 900                         batch_copy[vq->batch_copy_nb_elems].src =
 901                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
 902                         batch_copy[vq->batch_copy_nb_elems].log_addr =
 903                                 buf_iova + buf_offset;
 904                         batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 905                         vq->batch_copy_nb_elems++;
 906                 }
 907
 908                 mbuf_avail  -= cpy_len;
 909                 mbuf_offset += cpy_len;
 910                 buf_avail  -= cpy_len;
 911                 buf_offset += cpy_len;
 912         }
 913
 914 out:
 915
 916         return error;
 917 }
 918
 919 static __rte_always_inline void
 920 async_fill_vec(struct iovec *v, void *base, size_t len)
 921 {
 922         v->iov_base = base;
 923         v->iov_len = len;
 924 }
 925
 926 static __rte_always_inline void
 927 async_fill_iter(struct rte_vhost_iov_iter *it, size_t count,
 928         struct iovec *vec, unsigned long nr_seg)
 929 {
 930         it->offset = 0;
 931         it->count = count;
 932
 933         if (count) {
 934                 it->iov = vec;
 935                 it->nr_segs = nr_seg;
 936         } else {
 937                 it->iov = 0;
 938                 it->nr_segs = 0;
 939         }
 940 }
 941
 942 static __rte_always_inline void
 943 async_fill_desc(struct rte_vhost_async_desc *desc,
 944         struct rte_vhost_iov_iter *src, struct rte_vhost_iov_iter *dst)
 945 {
 946         desc->src = src;
 947         desc->dst = dst;
 948 }
 949
 950 static __rte_always_inline int
 951 async_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 952                         struct rte_mbuf *m, struct buf_vector *buf_vec,
 953                         uint16_t nr_vec, uint16_t num_buffers,
 954                         struct iovec *src_iovec, struct iovec *dst_iovec,
 955                         struct rte_vhost_iov_iter *src_it,
 956                         struct rte_vhost_iov_iter *dst_it)
 957 {
 958         uint32_t vec_idx = 0;
 959         uint32_t mbuf_offset, mbuf_avail;
 960         uint32_t buf_offset, buf_avail;
 961         uint64_t buf_addr, buf_iova, buf_len;
 962         uint32_t cpy_len, cpy_threshold;
 963         uint64_t hdr_addr;
 964         struct rte_mbuf *hdr_mbuf;
 965         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 966         struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
 967         int error = 0;
 968         uint64_t mapped_len;
 969
 970         uint32_t tlen = 0;
 971         int tvec_idx = 0;
 972         void *hpa;
 973
 974         if (unlikely(m == NULL)) {
 975                 error = -1;
 976                 goto out;
 977         }
 978
 979         cpy_threshold = vq->async_threshold;
 980
 981         buf_addr = buf_vec[vec_idx].buf_addr;
 982         buf_iova = buf_vec[vec_idx].buf_iova;
 983         buf_len = buf_vec[vec_idx].buf_len;
 984
 985         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 986                 error = -1;
 987                 goto out;
 988         }
 989
 990         hdr_mbuf = m;
 991         hdr_addr = buf_addr;
 992         if (unlikely(buf_len < dev->vhost_hlen)) {
 993                 memset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf));
 994                 hdr = &tmp_hdr;
 995         } else
 996                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
 997
 998         VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n",
 999                 dev->vid, num_buffers);
1000
1001         if (unlikely(buf_len < dev->vhost_hlen)) {
1002                 buf_offset = dev->vhost_hlen - buf_len;
1003                 vec_idx++;
1004                 buf_addr = buf_vec[vec_idx].buf_addr;
1005                 buf_iova = buf_vec[vec_idx].buf_iova;
1006                 buf_len = buf_vec[vec_idx].buf_len;
1007                 buf_avail = buf_len - buf_offset;
1008         } else {
1009                 buf_offset = dev->vhost_hlen;
1010                 buf_avail = buf_len - dev->vhost_hlen;
1011         }
1012
1013         mbuf_avail  = rte_pktmbuf_data_len(m);
1014         mbuf_offset = 0;
1015
1016         while (mbuf_avail != 0 || m->next != NULL) {
1017                 /* done with current buf, get the next one */
1018                 if (buf_avail == 0) {
1019                         vec_idx++;
1020                         if (unlikely(vec_idx >= nr_vec)) {
1021                                 error = -1;
1022                                 goto out;
1023                         }
1024
1025                         buf_addr = buf_vec[vec_idx].buf_addr;
1026                         buf_iova = buf_vec[vec_idx].buf_iova;
1027                         buf_len = buf_vec[vec_idx].buf_len;
1028
1029                         buf_offset = 0;
1030                         buf_avail  = buf_len;
1031                 }
1032
1033                 /* done with current mbuf, get the next one */
1034                 if (mbuf_avail == 0) {
1035                         m = m->next;
1036
1037                         mbuf_offset = 0;
1038                         mbuf_avail  = rte_pktmbuf_data_len(m);
1039                 }
1040
1041                 if (hdr_addr) {
1042                         virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
1043                         if (rxvq_is_mergeable(dev))
1044                                 ASSIGN_UNLESS_EQUAL(hdr->num_buffers,
1045                                                 num_buffers);
1046
1047                         if (unlikely(hdr == &tmp_hdr)) {
1048                                 copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr);
1049                         } else {
1050                                 PRINT_PACKET(dev, (uintptr_t)hdr_addr,
1051                                                 dev->vhost_hlen, 0);
1052                                 vhost_log_cache_write_iova(dev, vq,
1053                                                 buf_vec[0].buf_iova,
1054                                                 dev->vhost_hlen);
1055                         }
1056
1057                         hdr_addr = 0;
1058                 }
1059
1060                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
1061
1062                 while (unlikely(cpy_len && cpy_len >= cpy_threshold)) {
1063                         hpa = (void *)(uintptr_t)gpa_to_first_hpa(dev,
1064                                         buf_iova + buf_offset,
1065                                         cpy_len, &mapped_len);
1066
1067                         if (unlikely(!hpa || mapped_len < cpy_threshold))
1068                                 break;
1069
1070                         async_fill_vec(src_iovec + tvec_idx,
1071                                 (void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
1072                                 mbuf_offset), (size_t)mapped_len);
1073
1074                         async_fill_vec(dst_iovec + tvec_idx,
1075                                         hpa, (size_t)mapped_len);
1076
1077                         tlen += (uint32_t)mapped_len;
1078                         cpy_len -= (uint32_t)mapped_len;
1079                         mbuf_avail  -= (uint32_t)mapped_len;
1080                         mbuf_offset += (uint32_t)mapped_len;
1081                         buf_avail  -= (uint32_t)mapped_len;
1082                         buf_offset += (uint32_t)mapped_len;
1083                         tvec_idx++;
1084                 }
1085
1086                 if (likely(cpy_len)) {
1087                         if (unlikely(vq->batch_copy_nb_elems >= vq->size)) {
1088                                 rte_memcpy(
1089                                 (void *)((uintptr_t)(buf_addr + buf_offset)),
1090                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
1091                                 cpy_len);
1092
1093                                 PRINT_PACKET(dev,
1094                                         (uintptr_t)(buf_addr + buf_offset),
1095                                         cpy_len, 0);
1096                         } else {
1097                                 batch_copy[vq->batch_copy_nb_elems].dst =
1098                                 (void *)((uintptr_t)(buf_addr + buf_offset));
1099                                 batch_copy[vq->batch_copy_nb_elems].src =
1100                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
1101                                 batch_copy[vq->batch_copy_nb_elems].log_addr =
1102                                         buf_iova + buf_offset;
1103                                 batch_copy[vq->batch_copy_nb_elems].len =
1104                                         cpy_len;
1105                                 vq->batch_copy_nb_elems++;
1106                         }
1107
1108                         mbuf_avail  -= cpy_len;
1109                         mbuf_offset += cpy_len;
1110                         buf_avail  -= cpy_len;
1111                         buf_offset += cpy_len;
1112                 }
1113
1114         }
1115
1116 out:
1117         if (tlen) {
1118                 async_fill_iter(src_it, tlen, src_iovec, tvec_idx);
1119                 async_fill_iter(dst_it, tlen, dst_iovec, tvec_idx);
1120         } else {
1121                 src_it->count = 0;
1122         }
1123
1124         return error;
1125 }
1126
1127 static __rte_always_inline int
1128 vhost_enqueue_single_packed(struct virtio_net *dev,
1129                             struct vhost_virtqueue *vq,
1130                             struct rte_mbuf *pkt,
1131                             struct buf_vector *buf_vec,
1132                             uint16_t *nr_descs)
1133 {
1134         uint16_t nr_vec = 0;
1135         uint16_t avail_idx = vq->last_avail_idx;
1136         uint16_t max_tries, tries = 0;
1137         uint16_t buf_id = 0;
1138         uint32_t len = 0;
1139         uint16_t desc_count;
1140         uint32_t size = pkt->pkt_len + sizeof(struct virtio_net_hdr_mrg_rxbuf);
1141         uint16_t num_buffers = 0;
1142         uint32_t buffer_len[vq->size];
1143         uint16_t buffer_buf_id[vq->size];
1144         uint16_t buffer_desc_count[vq->size];
1145
1146         if (rxvq_is_mergeable(dev))
1147                 max_tries = vq->size - 1;
1148         else
1149                 max_tries = 1;
1150
1151         while (size > 0) {
1152                 /*
1153                  * if we tried all available ring items, and still
1154                  * can't get enough buf, it means something abnormal
1155                  * happened.
1156                  */
1157                 if (unlikely(++tries > max_tries))
1158                         return -1;
1159
1160                 if (unlikely(fill_vec_buf_packed(dev, vq,
1161                                                 avail_idx, &desc_count,
1162                                                 buf_vec, &nr_vec,
1163                                                 &buf_id, &len,
1164                                                 VHOST_ACCESS_RW) < 0))
1165                         return -1;
1166
1167                 len = RTE_MIN(len, size);
1168                 size -= len;
1169
1170                 buffer_len[num_buffers] = len;
1171                 buffer_buf_id[num_buffers] = buf_id;
1172                 buffer_desc_count[num_buffers] = desc_count;
1173                 num_buffers += 1;
1174
1175                 *nr_descs += desc_count;
1176                 avail_idx += desc_count;
1177                 if (avail_idx >= vq->size)
1178                         avail_idx -= vq->size;
1179         }
1180
1181         if (copy_mbuf_to_desc(dev, vq, pkt, buf_vec, nr_vec, num_buffers) < 0)
1182                 return -1;
1183
1184         vhost_shadow_enqueue_single_packed(dev, vq, buffer_len, buffer_buf_id,
1185                                            buffer_desc_count, num_buffers);
1186
1187         return 0;
1188 }
1189
1190 static __rte_noinline uint32_t
1191 virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
1192         struct rte_mbuf **pkts, uint32_t count)
1193 {
1194         uint32_t pkt_idx = 0;
1195         uint16_t num_buffers;
1196         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1197         uint16_t avail_head;
1198
1199         /*
1200          * The ordering between avail index and
1201          * desc reads needs to be enforced.
1202          */
1203         avail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE);
1204
1205         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
1206
1207         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
1208                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
1209                 uint16_t nr_vec = 0;
1210
1211                 if (unlikely(reserve_avail_buf_split(dev, vq,
1212                                                 pkt_len, buf_vec, &num_buffers,
1213                                                 avail_head, &nr_vec) < 0)) {
1214                         VHOST_LOG_DATA(DEBUG,
1215                                 "(%d) failed to get enough desc from vring\n",
1216                                 dev->vid);
1217                         vq->shadow_used_idx -= num_buffers;
1218                         break;
1219                 }
1220
1221                 VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n",
1222                         dev->vid, vq->last_avail_idx,
1223                         vq->last_avail_idx + num_buffers);
1224
1225                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
1226                                                 buf_vec, nr_vec,
1227                                                 num_buffers) < 0) {
1228                         vq->shadow_used_idx -= num_buffers;
1229                         break;
1230                 }
1231
1232                 vq->last_avail_idx += num_buffers;
1233         }
1234
1235         do_data_copy_enqueue(dev, vq);
1236
1237         if (likely(vq->shadow_used_idx)) {
1238                 flush_shadow_used_ring_split(dev, vq);
1239                 vhost_vring_call_split(dev, vq);
1240         }
1241
1242         return pkt_idx;
1243 }
1244
1245 static __rte_always_inline int
1246 virtio_dev_rx_batch_packed(struct virtio_net *dev,
1247                            struct vhost_virtqueue *vq,
1248                            struct rte_mbuf **pkts)
1249 {
1250         bool wrap_counter = vq->avail_wrap_counter;
1251         struct vring_packed_desc *descs = vq->desc_packed;
1252         uint16_t avail_idx = vq->last_avail_idx;
1253         uint64_t desc_addrs[PACKED_BATCH_SIZE];
1254         struct virtio_net_hdr_mrg_rxbuf *hdrs[PACKED_BATCH_SIZE];
1255         uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1256         uint64_t lens[PACKED_BATCH_SIZE];
1257         uint16_t ids[PACKED_BATCH_SIZE];
1258         uint16_t i;
1259
1260         if (unlikely(avail_idx & PACKED_BATCH_MASK))
1261                 return -1;
1262
1263         if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))
1264                 return -1;
1265
1266         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1267                 if (unlikely(pkts[i]->next != NULL))
1268                         return -1;
1269                 if (unlikely(!desc_is_avail(&descs[avail_idx + i],
1270                                             wrap_counter)))
1271                         return -1;
1272         }
1273
1274         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1275                 lens[i] = descs[avail_idx + i].len;
1276
1277         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1278                 if (unlikely(pkts[i]->pkt_len > (lens[i] - buf_offset)))
1279                         return -1;
1280         }
1281
1282         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1283                 desc_addrs[i] = vhost_iova_to_vva(dev, vq,
1284                                                   descs[avail_idx + i].addr,
1285                                                   &lens[i],
1286                                                   VHOST_ACCESS_RW);
1287
1288         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1289                 if (unlikely(!desc_addrs[i]))
1290                         return -1;
1291                 if (unlikely(lens[i] != descs[avail_idx + i].len))
1292                         return -1;
1293         }
1294
1295         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1296                 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]);
1297                 hdrs[i] = (struct virtio_net_hdr_mrg_rxbuf *)
1298                                         (uintptr_t)desc_addrs[i];
1299                 lens[i] = pkts[i]->pkt_len +
1300                         sizeof(struct virtio_net_hdr_mrg_rxbuf);
1301         }
1302
1303         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1304                 virtio_enqueue_offload(pkts[i], &hdrs[i]->hdr);
1305
1306         vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
1307
1308         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1309                 rte_memcpy((void *)(uintptr_t)(desc_addrs[i] + buf_offset),
1310                            rte_pktmbuf_mtod_offset(pkts[i], void *, 0),
1311                            pkts[i]->pkt_len);
1312         }
1313
1314         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1315                 vhost_log_cache_write_iova(dev, vq, descs[avail_idx + i].addr,
1316                                            lens[i]);
1317
1318         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1319                 ids[i] = descs[avail_idx + i].id;
1320
1321         vhost_flush_enqueue_batch_packed(dev, vq, lens, ids);
1322
1323         return 0;
1324 }
1325
1326 static __rte_always_inline int16_t
1327 virtio_dev_rx_single_packed(struct virtio_net *dev,
1328                             struct vhost_virtqueue *vq,
1329                             struct rte_mbuf *pkt)
1330 {
1331         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1332         uint16_t nr_descs = 0;
1333
1334         if (unlikely(vhost_enqueue_single_packed(dev, vq, pkt, buf_vec,
1335                                                  &nr_descs) < 0)) {
1336                 VHOST_LOG_DATA(DEBUG,
1337                                 "(%d) failed to get enough desc from vring\n",
1338                                 dev->vid);
1339                 return -1;
1340         }
1341
1342         VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n",
1343                         dev->vid, vq->last_avail_idx,
1344                         vq->last_avail_idx + nr_descs);
1345
1346         vq_inc_last_avail_packed(vq, nr_descs);
1347
1348         return 0;
1349 }
1350
1351 static __rte_noinline uint32_t
1352 virtio_dev_rx_packed(struct virtio_net *dev,
1353                      struct vhost_virtqueue *__rte_restrict vq,
1354                      struct rte_mbuf **__rte_restrict pkts,
1355                      uint32_t count)
1356 {
1357         uint32_t pkt_idx = 0;
1358
1359         do {
1360                 rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
1361
1362                 if (count - pkt_idx >= PACKED_BATCH_SIZE) {
1363                         if (!virtio_dev_rx_batch_packed(dev, vq,
1364                                                         &pkts[pkt_idx])) {
1365                                 pkt_idx += PACKED_BATCH_SIZE;
1366                                 continue;
1367                         }
1368                 }
1369
1370                 if (virtio_dev_rx_single_packed(dev, vq, pkts[pkt_idx]))
1371                         break;
1372                 pkt_idx++;
1373
1374         } while (pkt_idx < count);
1375
1376         if (vq->shadow_used_idx) {
1377                 do_data_copy_enqueue(dev, vq);
1378                 vhost_flush_enqueue_shadow_packed(dev, vq);
1379         }
1380
1381         if (pkt_idx)
1382                 vhost_vring_call_packed(dev, vq);
1383
1384         return pkt_idx;
1385 }
1386
1387 static __rte_always_inline uint32_t
1388 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
1389         struct rte_mbuf **pkts, uint32_t count)
1390 {
1391         struct vhost_virtqueue *vq;
1392         uint32_t nb_tx = 0;
1393
1394         VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
1395         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
1396                 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
1397                         dev->vid, __func__, queue_id);
1398                 return 0;
1399         }
1400
1401         vq = dev->virtqueue[queue_id];
1402
1403         rte_spinlock_lock(&vq->access_lock);
1404
1405         if (unlikely(!vq->enabled))
1406                 goto out_access_unlock;
1407
1408         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1409                 vhost_user_iotlb_rd_lock(vq);
1410
1411         if (unlikely(!vq->access_ok))
1412                 if (unlikely(vring_translate(dev, vq) < 0))
1413                         goto out;
1414
1415         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
1416         if (count == 0)
1417                 goto out;
1418
1419         if (vq_is_packed(dev))
1420                 nb_tx = virtio_dev_rx_packed(dev, vq, pkts, count);
1421         else
1422                 nb_tx = virtio_dev_rx_split(dev, vq, pkts, count);
1423
1424 out:
1425         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1426                 vhost_user_iotlb_rd_unlock(vq);
1427
1428 out_access_unlock:
1429         rte_spinlock_unlock(&vq->access_lock);
1430
1431         return nb_tx;
1432 }
1433
1434 uint16_t
1435 rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
1436         struct rte_mbuf **__rte_restrict pkts, uint16_t count)
1437 {
1438         struct virtio_net *dev = get_device(vid);
1439
1440         if (!dev)
1441                 return 0;
1442
1443         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
1444                 VHOST_LOG_DATA(ERR,
1445                         "(%d) %s: built-in vhost net backend is disabled.\n",
1446                         dev->vid, __func__);
1447                 return 0;
1448         }
1449
1450         return virtio_dev_rx(dev, queue_id, pkts, count);
1451 }
1452
1453 static __rte_always_inline uint16_t
1454 virtio_dev_rx_async_get_info_idx(uint16_t pkts_idx,
1455         uint16_t vq_size, uint16_t n_inflight)
1456 {
1457         return pkts_idx > n_inflight ? (pkts_idx - n_inflight) :
1458                 (vq_size - n_inflight + pkts_idx) & (vq_size - 1);
1459 }
1460
1461 static __rte_always_inline void
1462 store_dma_desc_info_split(struct vring_used_elem *s_ring, struct vring_used_elem *d_ring,
1463                 uint16_t ring_size, uint16_t s_idx, uint16_t d_idx, uint16_t count)
1464 {
1465         uint16_t elem_size = sizeof(struct vring_used_elem);
1466
1467         if (d_idx + count <= ring_size) {
1468                 rte_memcpy(d_ring + d_idx, s_ring + s_idx, count * elem_size);
1469         } else {
1470                 uint16_t size = ring_size - d_idx;
1471
1472                 rte_memcpy(d_ring + d_idx, s_ring + s_idx, size * elem_size);
1473                 rte_memcpy(d_ring, s_ring + s_idx + size, (count - size) * elem_size);
1474         }
1475 }
1476
1477 static __rte_noinline uint32_t
1478 virtio_dev_rx_async_submit_split(struct virtio_net *dev,
1479         struct vhost_virtqueue *vq, uint16_t queue_id,
1480         struct rte_mbuf **pkts, uint32_t count,
1481         struct rte_mbuf **comp_pkts, uint32_t *comp_count)
1482 {
1483         uint32_t pkt_idx = 0, pkt_burst_idx = 0;
1484         uint16_t num_buffers;
1485         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1486         uint16_t avail_head;
1487
1488         struct rte_vhost_iov_iter *it_pool = vq->it_pool;
1489         struct iovec *vec_pool = vq->vec_pool;
1490         struct rte_vhost_async_desc tdes[MAX_PKT_BURST];
1491         struct iovec *src_iovec = vec_pool;
1492         struct iovec *dst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >> 1);
1493         uint16_t slot_idx = 0;
1494         uint16_t segs_await = 0;
1495         uint16_t iovec_idx = 0, it_idx = 0;
1496         struct async_inflight_info *pkts_info = vq->async_pkts_info;
1497         uint32_t n_pkts = 0, pkt_err = 0;
1498         uint32_t num_async_pkts = 0, num_done_pkts = 0;
1499         struct {
1500                 uint16_t pkt_idx;
1501                 uint16_t last_avail_idx;
1502         } async_pkts_log[MAX_PKT_BURST];
1503
1504         /*
1505          * The ordering between avail index and desc reads need to be enforced.
1506          */
1507         avail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE);
1508
1509         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
1510
1511         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
1512                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
1513                 uint16_t nr_vec = 0;
1514
1515                 if (unlikely(reserve_avail_buf_split(dev, vq,
1516                                                 pkt_len, buf_vec, &num_buffers,
1517                                                 avail_head, &nr_vec) < 0)) {
1518                         VHOST_LOG_DATA(DEBUG,
1519                                 "(%d) failed to get enough desc from vring\n",
1520                                 dev->vid);
1521                         vq->shadow_used_idx -= num_buffers;
1522                         break;
1523                 }
1524
1525                 VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n",
1526                         dev->vid, vq->last_avail_idx,
1527                         vq->last_avail_idx + num_buffers);
1528
1529                 if (async_mbuf_to_desc(dev, vq, pkts[pkt_idx], buf_vec, nr_vec, num_buffers,
1530                                 &src_iovec[iovec_idx], &dst_iovec[iovec_idx],
1531                                 &it_pool[it_idx], &it_pool[it_idx + 1]) < 0) {
1532                         vq->shadow_used_idx -= num_buffers;
1533                         break;
1534                 }
1535
1536                 slot_idx = (vq->async_pkts_idx + num_async_pkts) &
1537                         (vq->size - 1);
1538                 if (it_pool[it_idx].count) {
1539                         uint16_t from, to;
1540
1541                         async_fill_desc(&tdes[pkt_burst_idx++],
1542                                 &it_pool[it_idx], &it_pool[it_idx + 1]);
1543                         pkts_info[slot_idx].descs = num_buffers;
1544                         pkts_info[slot_idx].mbuf = pkts[pkt_idx];
1545                         async_pkts_log[num_async_pkts].pkt_idx = pkt_idx;
1546                         async_pkts_log[num_async_pkts++].last_avail_idx =
1547                                 vq->last_avail_idx;
1548
1549                         iovec_idx += it_pool[it_idx].nr_segs;
1550                         it_idx += 2;
1551
1552                         segs_await += it_pool[it_idx].nr_segs;
1553
1554                         /**
1555                          * recover shadow used ring and keep DMA-occupied
1556                          * descriptors.
1557                          */
1558                         from = vq->shadow_used_idx - num_buffers;
1559                         to = vq->async_desc_idx & (vq->size - 1);
1560
1561                         store_dma_desc_info_split(vq->shadow_used_split,
1562                                         vq->async_descs_split, vq->size, from, to, num_buffers);
1563
1564                         vq->async_desc_idx += num_buffers;
1565                         vq->shadow_used_idx -= num_buffers;
1566                 } else
1567                         comp_pkts[num_done_pkts++] = pkts[pkt_idx];
1568
1569                 vq->last_avail_idx += num_buffers;
1570
1571                 /*
1572                  * conditions to trigger async device transfer:
1573                  * - buffered packet number reaches transfer threshold
1574                  * - unused async iov number is less than max vhost vector
1575                  */
1576                 if (unlikely(pkt_burst_idx >= VHOST_ASYNC_BATCH_THRESHOLD ||
1577                         ((VHOST_MAX_ASYNC_VEC >> 1) - segs_await <
1578                         BUF_VECTOR_MAX))) {
1579                         n_pkts = vq->async_ops.transfer_data(dev->vid,
1580                                         queue_id, tdes, 0, pkt_burst_idx);
1581                         iovec_idx = 0;
1582                         it_idx = 0;
1583
1584                         segs_await = 0;
1585                         vq->async_pkts_inflight_n += n_pkts;
1586
1587                         if (unlikely(n_pkts < pkt_burst_idx)) {
1588                                 /*
1589                                  * log error packets number here and do actual
1590                                  * error processing when applications poll
1591                                  * completion
1592                                  */
1593                                 pkt_err = pkt_burst_idx - n_pkts;
1594                                 pkt_burst_idx = 0;
1595                                 break;
1596                         }
1597
1598                         pkt_burst_idx = 0;
1599                 }
1600         }
1601
1602         if (pkt_burst_idx) {
1603                 n_pkts = vq->async_ops.transfer_data(dev->vid,
1604                                 queue_id, tdes, 0, pkt_burst_idx);
1605                 vq->async_pkts_inflight_n += n_pkts;
1606
1607                 if (unlikely(n_pkts < pkt_burst_idx))
1608                         pkt_err = pkt_burst_idx - n_pkts;
1609         }
1610
1611         do_data_copy_enqueue(dev, vq);
1612
1613         if (unlikely(pkt_err)) {
1614                 uint16_t num_descs = 0;
1615
1616                 num_async_pkts -= pkt_err;
1617                 /* calculate the sum of descriptors of DMA-error packets. */
1618                 while (pkt_err-- > 0) {
1619                         num_descs += pkts_info[slot_idx & (vq->size - 1)].descs;
1620                         slot_idx--;
1621                 }
1622                 vq->async_desc_idx -= num_descs;
1623                 /* recover shadow used ring and available ring */
1624                 vq->shadow_used_idx -= (vq->last_avail_idx -
1625                                 async_pkts_log[num_async_pkts].last_avail_idx -
1626                                 num_descs);
1627                 vq->last_avail_idx =
1628                         async_pkts_log[num_async_pkts].last_avail_idx;
1629                 pkt_idx = async_pkts_log[num_async_pkts].pkt_idx;
1630                 num_done_pkts = pkt_idx - num_async_pkts;
1631         }
1632
1633         vq->async_pkts_idx += num_async_pkts;
1634         *comp_count = num_done_pkts;
1635
1636         if (likely(vq->shadow_used_idx)) {
1637                 flush_shadow_used_ring_split(dev, vq);
1638                 vhost_vring_call_split(dev, vq);
1639         }
1640
1641         return pkt_idx;
1642 }
1643
1644 static __rte_always_inline void
1645 write_back_completed_descs_split(struct vhost_virtqueue *vq, uint16_t n_descs)
1646 {
1647         uint16_t nr_left = n_descs;
1648         uint16_t nr_copy;
1649         uint16_t to, from;
1650
1651         do {
1652                 from = vq->last_async_desc_idx & (vq->size - 1);
1653                 nr_copy = nr_left + from <= vq->size ? nr_left : vq->size - from;
1654                 to = vq->last_used_idx & (vq->size - 1);
1655
1656                 if (to + nr_copy <= vq->size) {
1657                         rte_memcpy(&vq->used->ring[to], &vq->async_descs_split[from],
1658                                         nr_copy * sizeof(struct vring_used_elem));
1659                 } else {
1660                         uint16_t size = vq->size - to;
1661
1662                         rte_memcpy(&vq->used->ring[to], &vq->async_descs_split[from],
1663                                         size * sizeof(struct vring_used_elem));
1664                         rte_memcpy(&vq->used->ring[0], &vq->async_descs_split[from + size],
1665                                         (nr_copy - size) * sizeof(struct vring_used_elem));
1666                 }
1667
1668                 vq->last_async_desc_idx += nr_copy;
1669                 vq->last_used_idx += nr_copy;
1670                 nr_left -= nr_copy;
1671         } while (nr_left > 0);
1672 }
1673
1674 uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id,
1675                 struct rte_mbuf **pkts, uint16_t count)
1676 {
1677         struct virtio_net *dev = get_device(vid);
1678         struct vhost_virtqueue *vq;
1679         uint16_t n_pkts_cpl = 0, n_pkts_put = 0, n_descs = 0;
1680         uint16_t start_idx, pkts_idx, vq_size;
1681         struct async_inflight_info *pkts_info;
1682         uint16_t from, i;
1683
1684         if (!dev)
1685                 return 0;
1686
1687         VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
1688         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
1689                 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
1690                         dev->vid, __func__, queue_id);
1691                 return 0;
1692         }
1693
1694         vq = dev->virtqueue[queue_id];
1695
1696         if (unlikely(!vq->async_registered)) {
1697                 VHOST_LOG_DATA(ERR, "(%d) %s: async not registered for queue id %d.\n",
1698                         dev->vid, __func__, queue_id);
1699                 return 0;
1700         }
1701
1702         rte_spinlock_lock(&vq->access_lock);
1703
1704         pkts_idx = vq->async_pkts_idx & (vq->size - 1);
1705         pkts_info = vq->async_pkts_info;
1706         vq_size = vq->size;
1707         start_idx = virtio_dev_rx_async_get_info_idx(pkts_idx,
1708                 vq_size, vq->async_pkts_inflight_n);
1709
1710         if (count > vq->async_last_pkts_n)
1711                 n_pkts_cpl = vq->async_ops.check_completed_copies(vid,
1712                         queue_id, 0, count - vq->async_last_pkts_n);
1713         n_pkts_cpl += vq->async_last_pkts_n;
1714
1715         n_pkts_put = RTE_MIN(count, n_pkts_cpl);
1716         if (unlikely(n_pkts_put == 0)) {
1717                 vq->async_last_pkts_n = n_pkts_cpl;
1718                 goto done;
1719         }
1720
1721         for (i = 0; i < n_pkts_put; i++) {
1722                 from = (start_idx + i) & (vq_size - 1);
1723                 n_descs += pkts_info[from].descs;
1724                 pkts[i] = pkts_info[from].mbuf;
1725         }
1726         vq->async_last_pkts_n = n_pkts_cpl - n_pkts_put;
1727         vq->async_pkts_inflight_n -= n_pkts_put;
1728
1729         if (likely(vq->enabled && vq->access_ok)) {
1730                 write_back_completed_descs_split(vq, n_descs);
1731
1732                 __atomic_add_fetch(&vq->used->idx, n_descs, __ATOMIC_RELEASE);
1733                 vhost_vring_call_split(dev, vq);
1734         } else
1735                 vq->last_async_desc_idx += n_descs;
1736
1737 done:
1738         rte_spinlock_unlock(&vq->access_lock);
1739
1740         return n_pkts_put;
1741 }
1742
1743 static __rte_always_inline uint32_t
1744 virtio_dev_rx_async_submit(struct virtio_net *dev, uint16_t queue_id,
1745         struct rte_mbuf **pkts, uint32_t count,
1746         struct rte_mbuf **comp_pkts, uint32_t *comp_count)
1747 {
1748         struct vhost_virtqueue *vq;
1749         uint32_t nb_tx = 0;
1750
1751         VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
1752         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
1753                 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
1754                         dev->vid, __func__, queue_id);
1755                 return 0;
1756         }
1757
1758         vq = dev->virtqueue[queue_id];
1759
1760         rte_spinlock_lock(&vq->access_lock);
1761
1762         if (unlikely(!vq->enabled || !vq->async_registered))
1763                 goto out_access_unlock;
1764
1765         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1766                 vhost_user_iotlb_rd_lock(vq);
1767
1768         if (unlikely(!vq->access_ok))
1769                 if (unlikely(vring_translate(dev, vq) < 0))
1770                         goto out;
1771
1772         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
1773         if (count == 0)
1774                 goto out;
1775
1776         /* TODO: packed queue not implemented */
1777         if (vq_is_packed(dev))
1778                 nb_tx = 0;
1779         else
1780                 nb_tx = virtio_dev_rx_async_submit_split(dev,
1781                                 vq, queue_id, pkts, count, comp_pkts,
1782                                 comp_count);
1783
1784 out:
1785         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1786                 vhost_user_iotlb_rd_unlock(vq);
1787
1788 out_access_unlock:
1789         rte_spinlock_unlock(&vq->access_lock);
1790
1791         return nb_tx;
1792 }
1793
1794 uint16_t
1795 rte_vhost_submit_enqueue_burst(int vid, uint16_t queue_id,
1796                 struct rte_mbuf **pkts, uint16_t count,
1797                 struct rte_mbuf **comp_pkts, uint32_t *comp_count)
1798 {
1799         struct virtio_net *dev = get_device(vid);
1800
1801         *comp_count = 0;
1802         if (!dev)
1803                 return 0;
1804
1805         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
1806                 VHOST_LOG_DATA(ERR,
1807                         "(%d) %s: built-in vhost net backend is disabled.\n",
1808                         dev->vid, __func__);
1809                 return 0;
1810         }
1811
1812         return virtio_dev_rx_async_submit(dev, queue_id, pkts, count, comp_pkts,
1813                         comp_count);
1814 }
1815
1816 static inline bool
1817 virtio_net_with_host_offload(struct virtio_net *dev)
1818 {
1819         if (dev->features &
1820                         ((1ULL << VIRTIO_NET_F_CSUM) |
1821                          (1ULL << VIRTIO_NET_F_HOST_ECN) |
1822                          (1ULL << VIRTIO_NET_F_HOST_TSO4) |
1823                          (1ULL << VIRTIO_NET_F_HOST_TSO6) |
1824                          (1ULL << VIRTIO_NET_F_HOST_UFO)))
1825                 return true;
1826
1827         return false;
1828 }
1829
1830 static void
1831 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
1832 {
1833         struct rte_ipv4_hdr *ipv4_hdr;
1834         struct rte_ipv6_hdr *ipv6_hdr;
1835         void *l3_hdr = NULL;
1836         struct rte_ether_hdr *eth_hdr;
1837         uint16_t ethertype;
1838
1839         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
1840
1841         m->l2_len = sizeof(struct rte_ether_hdr);
1842         ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
1843
1844         if (ethertype == RTE_ETHER_TYPE_VLAN) {
1845                 struct rte_vlan_hdr *vlan_hdr =
1846                         (struct rte_vlan_hdr *)(eth_hdr + 1);
1847
1848                 m->l2_len += sizeof(struct rte_vlan_hdr);
1849                 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
1850         }
1851
1852         l3_hdr = (char *)eth_hdr + m->l2_len;
1853
1854         switch (ethertype) {
1855         case RTE_ETHER_TYPE_IPV4:
1856                 ipv4_hdr = l3_hdr;
1857                 *l4_proto = ipv4_hdr->next_proto_id;
1858                 m->l3_len = rte_ipv4_hdr_len(ipv4_hdr);
1859                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1860                 m->ol_flags |= PKT_TX_IPV4;
1861                 break;
1862         case RTE_ETHER_TYPE_IPV6:
1863                 ipv6_hdr = l3_hdr;
1864                 *l4_proto = ipv6_hdr->proto;
1865                 m->l3_len = sizeof(struct rte_ipv6_hdr);
1866                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1867                 m->ol_flags |= PKT_TX_IPV6;
1868                 break;
1869         default:
1870                 m->l3_len = 0;
1871                 *l4_proto = 0;
1872                 *l4_hdr = NULL;
1873                 break;
1874         }
1875 }
1876
1877 static __rte_always_inline void
1878 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
1879 {
1880         uint16_t l4_proto = 0;
1881         void *l4_hdr = NULL;
1882         struct rte_tcp_hdr *tcp_hdr = NULL;
1883
1884         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
1885                 return;
1886
1887         parse_ethernet(m, &l4_proto, &l4_hdr);
1888         if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1889                 if (hdr->csum_start == (m->l2_len + m->l3_len)) {
1890                         switch (hdr->csum_offset) {
1891                         case (offsetof(struct rte_tcp_hdr, cksum)):
1892                                 if (l4_proto == IPPROTO_TCP)
1893                                         m->ol_flags |= PKT_TX_TCP_CKSUM;
1894                                 break;
1895                         case (offsetof(struct rte_udp_hdr, dgram_cksum)):
1896                                 if (l4_proto == IPPROTO_UDP)
1897                                         m->ol_flags |= PKT_TX_UDP_CKSUM;
1898                                 break;
1899                         case (offsetof(struct rte_sctp_hdr, cksum)):
1900                                 if (l4_proto == IPPROTO_SCTP)
1901                                         m->ol_flags |= PKT_TX_SCTP_CKSUM;
1902                                 break;
1903                         default:
1904                                 break;
1905                         }
1906                 }
1907         }
1908
1909         if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1910                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1911                 case VIRTIO_NET_HDR_GSO_TCPV4:
1912                 case VIRTIO_NET_HDR_GSO_TCPV6:
1913                         tcp_hdr = l4_hdr;
1914                         m->ol_flags |= PKT_TX_TCP_SEG;
1915                         m->tso_segsz = hdr->gso_size;
1916                         m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
1917                         break;
1918                 case VIRTIO_NET_HDR_GSO_UDP:
1919                         m->ol_flags |= PKT_TX_UDP_SEG;
1920                         m->tso_segsz = hdr->gso_size;
1921                         m->l4_len = sizeof(struct rte_udp_hdr);
1922                         break;
1923                 default:
1924                         VHOST_LOG_DATA(WARNING,
1925                                 "unsupported gso type %u.\n", hdr->gso_type);
1926                         break;
1927                 }
1928         }
1929 }
1930
1931 static __rte_noinline void
1932 copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr,
1933                 struct buf_vector *buf_vec)
1934 {
1935         uint64_t len;
1936         uint64_t remain = sizeof(struct virtio_net_hdr);
1937         uint64_t src;
1938         uint64_t dst = (uint64_t)(uintptr_t)hdr;
1939
1940         while (remain) {
1941                 len = RTE_MIN(remain, buf_vec->buf_len);
1942                 src = buf_vec->buf_addr;
1943                 rte_memcpy((void *)(uintptr_t)dst,
1944                                 (void *)(uintptr_t)src, len);
1945
1946                 remain -= len;
1947                 dst += len;
1948                 buf_vec++;
1949         }
1950 }
1951
1952 static __rte_always_inline int
1953 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
1954                   struct buf_vector *buf_vec, uint16_t nr_vec,
1955                   struct rte_mbuf *m, struct rte_mempool *mbuf_pool)
1956 {
1957         uint32_t buf_avail, buf_offset;
1958         uint64_t buf_addr, buf_len;
1959         uint32_t mbuf_avail, mbuf_offset;
1960         uint32_t cpy_len;
1961         struct rte_mbuf *cur = m, *prev = m;
1962         struct virtio_net_hdr tmp_hdr;
1963         struct virtio_net_hdr *hdr = NULL;
1964         /* A counter to avoid desc dead loop chain */
1965         uint16_t vec_idx = 0;
1966         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
1967         int error = 0;
1968
1969         buf_addr = buf_vec[vec_idx].buf_addr;
1970         buf_len = buf_vec[vec_idx].buf_len;
1971
1972         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
1973                 error = -1;
1974                 goto out;
1975         }
1976
1977         if (virtio_net_with_host_offload(dev)) {
1978                 if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
1979                         /*
1980                          * No luck, the virtio-net header doesn't fit
1981                          * in a contiguous virtual area.
1982                          */
1983                         copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec);
1984                         hdr = &tmp_hdr;
1985                 } else {
1986                         hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);
1987                 }
1988         }
1989
1990         /*
1991          * A virtio driver normally uses at least 2 desc buffers
1992          * for Tx: the first for storing the header, and others
1993          * for storing the data.
1994          */
1995         if (unlikely(buf_len < dev->vhost_hlen)) {
1996                 buf_offset = dev->vhost_hlen - buf_len;
1997                 vec_idx++;
1998                 buf_addr = buf_vec[vec_idx].buf_addr;
1999                 buf_len = buf_vec[vec_idx].buf_len;
2000                 buf_avail  = buf_len - buf_offset;
2001         } else if (buf_len == dev->vhost_hlen) {
2002                 if (unlikely(++vec_idx >= nr_vec))
2003                         goto out;
2004                 buf_addr = buf_vec[vec_idx].buf_addr;
2005                 buf_len = buf_vec[vec_idx].buf_len;
2006
2007                 buf_offset = 0;
2008                 buf_avail = buf_len;
2009         } else {
2010                 buf_offset = dev->vhost_hlen;
2011                 buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
2012         }
2013
2014         PRINT_PACKET(dev,
2015                         (uintptr_t)(buf_addr + buf_offset),
2016                         (uint32_t)buf_avail, 0);
2017
2018         mbuf_offset = 0;
2019         mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
2020         while (1) {
2021                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
2022
2023                 if (likely(cpy_len > MAX_BATCH_LEN ||
2024                                         vq->batch_copy_nb_elems >= vq->size ||
2025                                         (hdr && cur == m))) {
2026                         rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
2027                                                 mbuf_offset),
2028                                         (void *)((uintptr_t)(buf_addr +
2029                                                         buf_offset)), cpy_len);
2030                 } else {
2031                         batch_copy[vq->batch_copy_nb_elems].dst =
2032                                 rte_pktmbuf_mtod_offset(cur, void *,
2033                                                 mbuf_offset);
2034                         batch_copy[vq->batch_copy_nb_elems].src =
2035                                 (void *)((uintptr_t)(buf_addr + buf_offset));
2036                         batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
2037                         vq->batch_copy_nb_elems++;
2038                 }
2039
2040                 mbuf_avail  -= cpy_len;
2041                 mbuf_offset += cpy_len;
2042                 buf_avail -= cpy_len;
2043                 buf_offset += cpy_len;
2044
2045                 /* This buf reaches to its end, get the next one */
2046                 if (buf_avail == 0) {
2047                         if (++vec_idx >= nr_vec)
2048                                 break;
2049
2050                         buf_addr = buf_vec[vec_idx].buf_addr;
2051                         buf_len = buf_vec[vec_idx].buf_len;
2052
2053                         buf_offset = 0;
2054                         buf_avail  = buf_len;
2055
2056                         PRINT_PACKET(dev, (uintptr_t)buf_addr,
2057                                         (uint32_t)buf_avail, 0);
2058                 }
2059
2060                 /*
2061                  * This mbuf reaches to its end, get a new one
2062                  * to hold more data.
2063                  */
2064                 if (mbuf_avail == 0) {
2065                         cur = rte_pktmbuf_alloc(mbuf_pool);
2066                         if (unlikely(cur == NULL)) {
2067                                 VHOST_LOG_DATA(ERR, "Failed to "
2068                                         "allocate memory for mbuf.\n");
2069                                 error = -1;
2070                                 goto out;
2071                         }
2072
2073                         prev->next = cur;
2074                         prev->data_len = mbuf_offset;
2075                         m->nb_segs += 1;
2076                         m->pkt_len += mbuf_offset;
2077                         prev = cur;
2078
2079                         mbuf_offset = 0;
2080                         mbuf_avail  = cur->buf_len - RTE_PKTMBUF_HEADROOM;
2081                 }
2082         }
2083
2084         prev->data_len = mbuf_offset;
2085         m->pkt_len    += mbuf_offset;
2086
2087         if (hdr)
2088                 vhost_dequeue_offload(hdr, m);
2089
2090 out:
2091
2092         return error;
2093 }
2094
2095 static void
2096 virtio_dev_extbuf_free(void *addr __rte_unused, void *opaque)
2097 {
2098         rte_free(opaque);
2099 }
2100
2101 static int
2102 virtio_dev_extbuf_alloc(struct rte_mbuf *pkt, uint32_t size)
2103 {
2104         struct rte_mbuf_ext_shared_info *shinfo = NULL;
2105         uint32_t total_len = RTE_PKTMBUF_HEADROOM + size;
2106         uint16_t buf_len;
2107         rte_iova_t iova;
2108         void *buf;
2109
2110         total_len += sizeof(*shinfo) + sizeof(uintptr_t);
2111         total_len = RTE_ALIGN_CEIL(total_len, sizeof(uintptr_t));
2112
2113         if (unlikely(total_len > UINT16_MAX))
2114                 return -ENOSPC;
2115
2116         buf_len = total_len;
2117         buf = rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE);
2118         if (unlikely(buf == NULL))
2119                 return -ENOMEM;
2120
2121         /* Initialize shinfo */
2122         shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len,
2123                                                 virtio_dev_extbuf_free, buf);
2124         if (unlikely(shinfo == NULL)) {
2125                 rte_free(buf);
2126                 VHOST_LOG_DATA(ERR, "Failed to init shinfo\n");
2127                 return -1;
2128         }
2129
2130         iova = rte_malloc_virt2iova(buf);
2131         rte_pktmbuf_attach_extbuf(pkt, buf, iova, buf_len, shinfo);
2132         rte_pktmbuf_reset_headroom(pkt);
2133
2134         return 0;
2135 }
2136
2137 static __rte_always_inline int
2138 virtio_dev_pktmbuf_prep(struct virtio_net *dev, struct rte_mbuf *pkt,
2139                          uint32_t data_len)
2140 {
2141         if (rte_pktmbuf_tailroom(pkt) >= data_len)
2142                 return 0;
2143
2144         /* attach an external buffer if supported */
2145         if (dev->extbuf && !virtio_dev_extbuf_alloc(pkt, data_len))
2146                 return 0;
2147
2148         /* check if chained buffers are allowed */
2149         if (!dev->linearbuf)
2150                 return 0;
2151
2152         return -1;
2153 }
2154
2155 /*
2156  * Allocate a host supported pktmbuf.
2157  */
2158 static __rte_always_inline struct rte_mbuf *
2159 virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct rte_mempool *mp,
2160                          uint32_t data_len)
2161 {
2162         struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp);
2163
2164         if (unlikely(pkt == NULL)) {
2165                 VHOST_LOG_DATA(ERR,
2166                         "Failed to allocate memory for mbuf.\n");
2167                 return NULL;
2168         }
2169
2170         if (virtio_dev_pktmbuf_prep(dev, pkt, data_len)) {
2171                 /* Data doesn't fit into the buffer and the host supports
2172                  * only linear buffers
2173                  */
2174                 rte_pktmbuf_free(pkt);
2175                 return NULL;
2176         }
2177
2178         return pkt;
2179 }
2180
2181 static __rte_noinline uint16_t
2182 virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
2183         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
2184 {
2185         uint16_t i;
2186         uint16_t free_entries;
2187         uint16_t dropped = 0;
2188         static bool allocerr_warned;
2189
2190         /*
2191          * The ordering between avail index and
2192          * desc reads needs to be enforced.
2193          */
2194         free_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) -
2195                         vq->last_avail_idx;
2196         if (free_entries == 0)
2197                 return 0;
2198
2199         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
2200
2201         VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
2202
2203         count = RTE_MIN(count, MAX_PKT_BURST);
2204         count = RTE_MIN(count, free_entries);
2205         VHOST_LOG_DATA(DEBUG, "(%d) about to dequeue %u buffers\n",
2206                         dev->vid, count);
2207
2208         for (i = 0; i < count; i++) {
2209                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
2210                 uint16_t head_idx;
2211                 uint32_t buf_len;
2212                 uint16_t nr_vec = 0;
2213                 int err;
2214
2215                 if (unlikely(fill_vec_buf_split(dev, vq,
2216                                                 vq->last_avail_idx + i,
2217                                                 &nr_vec, buf_vec,
2218                                                 &head_idx, &buf_len,
2219                                                 VHOST_ACCESS_RO) < 0))
2220                         break;
2221
2222                 update_shadow_used_ring_split(vq, head_idx, 0);
2223
2224                 pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
2225                 if (unlikely(pkts[i] == NULL)) {
2226                         /*
2227                          * mbuf allocation fails for jumbo packets when external
2228                          * buffer allocation is not allowed and linear buffer
2229                          * is required. Drop this packet.
2230                          */
2231                         if (!allocerr_warned) {
2232                                 VHOST_LOG_DATA(ERR,
2233                                         "Failed mbuf alloc of size %d from %s on %s.\n",
2234                                         buf_len, mbuf_pool->name, dev->ifname);
2235                                 allocerr_warned = true;
2236                         }
2237                         dropped += 1;
2238                         i++;
2239                         break;
2240                 }
2241
2242                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
2243                                 mbuf_pool);
2244                 if (unlikely(err)) {
2245                         rte_pktmbuf_free(pkts[i]);
2246                         if (!allocerr_warned) {
2247                                 VHOST_LOG_DATA(ERR,
2248                                         "Failed to copy desc to mbuf on %s.\n",
2249                                         dev->ifname);
2250                                 allocerr_warned = true;
2251                         }
2252                         dropped += 1;
2253                         i++;
2254                         break;
2255                 }
2256         }
2257
2258         vq->last_avail_idx += i;
2259
2260         do_data_copy_dequeue(vq);
2261         if (unlikely(i < count))
2262                 vq->shadow_used_idx = i;
2263         if (likely(vq->shadow_used_idx)) {
2264                 flush_shadow_used_ring_split(dev, vq);
2265                 vhost_vring_call_split(dev, vq);
2266         }
2267
2268         return (i - dropped);
2269 }
2270
2271 static __rte_always_inline int
2272 vhost_reserve_avail_batch_packed(struct virtio_net *dev,
2273                                  struct vhost_virtqueue *vq,
2274                                  struct rte_mbuf **pkts,
2275                                  uint16_t avail_idx,
2276                                  uintptr_t *desc_addrs,
2277                                  uint16_t *ids)
2278 {
2279         bool wrap = vq->avail_wrap_counter;
2280         struct vring_packed_desc *descs = vq->desc_packed;
2281         uint64_t lens[PACKED_BATCH_SIZE];
2282         uint64_t buf_lens[PACKED_BATCH_SIZE];
2283         uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2284         uint16_t flags, i;
2285
2286         if (unlikely(avail_idx & PACKED_BATCH_MASK))
2287                 return -1;
2288         if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))
2289                 return -1;
2290
2291         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
2292                 flags = descs[avail_idx + i].flags;
2293                 if (unlikely((wrap != !!(flags & VRING_DESC_F_AVAIL)) ||
2294                              (wrap == !!(flags & VRING_DESC_F_USED))  ||
2295                              (flags & PACKED_DESC_SINGLE_DEQUEUE_FLAG)))
2296                         return -1;
2297         }
2298
2299         rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
2300
2301         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
2302                 lens[i] = descs[avail_idx + i].len;
2303
2304         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
2305                 desc_addrs[i] = vhost_iova_to_vva(dev, vq,
2306                                                   descs[avail_idx + i].addr,
2307                                                   &lens[i], VHOST_ACCESS_RW);
2308         }
2309
2310         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
2311                 if (unlikely(!desc_addrs[i]))
2312                         return -1;
2313                 if (unlikely((lens[i] != descs[avail_idx + i].len)))
2314                         return -1;
2315         }
2316
2317         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
2318                 if (virtio_dev_pktmbuf_prep(dev, pkts[i], lens[i]))
2319                         goto err;
2320         }
2321
2322         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
2323                 buf_lens[i] = pkts[i]->buf_len - pkts[i]->data_off;
2324
2325         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
2326                 if (unlikely(buf_lens[i] < (lens[i] - buf_offset)))
2327                         goto err;
2328         }
2329
2330         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
2331                 pkts[i]->pkt_len = lens[i] - buf_offset;
2332                 pkts[i]->data_len = pkts[i]->pkt_len;
2333                 ids[i] = descs[avail_idx + i].id;
2334         }
2335
2336         return 0;
2337
2338 err:
2339         return -1;
2340 }
2341
2342 static __rte_always_inline int
2343 virtio_dev_tx_batch_packed(struct virtio_net *dev,
2344                            struct vhost_virtqueue *vq,
2345                            struct rte_mbuf **pkts)
2346 {
2347         uint16_t avail_idx = vq->last_avail_idx;
2348         uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2349         struct virtio_net_hdr *hdr;
2350         uintptr_t desc_addrs[PACKED_BATCH_SIZE];
2351         uint16_t ids[PACKED_BATCH_SIZE];
2352         uint16_t i;
2353
2354         if (vhost_reserve_avail_batch_packed(dev, vq, pkts, avail_idx,
2355                                              desc_addrs, ids))
2356                 return -1;
2357
2358         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
2359                 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]);
2360
2361         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
2362                 rte_memcpy(rte_pktmbuf_mtod_offset(pkts[i], void *, 0),
2363                            (void *)(uintptr_t)(desc_addrs[i] + buf_offset),
2364                            pkts[i]->pkt_len);
2365
2366         if (virtio_net_with_host_offload(dev)) {
2367                 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
2368                         hdr = (struct virtio_net_hdr *)(desc_addrs[i]);
2369                         vhost_dequeue_offload(hdr, pkts[i]);
2370                 }
2371         }
2372
2373         if (virtio_net_is_inorder(dev))
2374                 vhost_shadow_dequeue_batch_packed_inorder(vq,
2375                         ids[PACKED_BATCH_SIZE - 1]);
2376         else
2377                 vhost_shadow_dequeue_batch_packed(dev, vq, ids);
2378
2379         vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
2380
2381         return 0;
2382 }
2383
2384 static __rte_always_inline int
2385 vhost_dequeue_single_packed(struct virtio_net *dev,
2386                             struct vhost_virtqueue *vq,
2387                             struct rte_mempool *mbuf_pool,
2388                             struct rte_mbuf *pkts,
2389                             uint16_t *buf_id,
2390                             uint16_t *desc_count)
2391 {
2392         struct buf_vector buf_vec[BUF_VECTOR_MAX];
2393         uint32_t buf_len;
2394         uint16_t nr_vec = 0;
2395         int err;
2396         static bool allocerr_warned;
2397
2398         if (unlikely(fill_vec_buf_packed(dev, vq,
2399                                          vq->last_avail_idx, desc_count,
2400                                          buf_vec, &nr_vec,
2401                                          buf_id, &buf_len,
2402                                          VHOST_ACCESS_RO) < 0))
2403                 return -1;
2404
2405         if (unlikely(virtio_dev_pktmbuf_prep(dev, pkts, buf_len))) {
2406                 if (!allocerr_warned) {
2407                         VHOST_LOG_DATA(ERR,
2408                                 "Failed mbuf alloc of size %d from %s on %s.\n",
2409                                 buf_len, mbuf_pool->name, dev->ifname);
2410                         allocerr_warned = true;
2411                 }
2412                 return -1;
2413         }
2414
2415         err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts,
2416                                 mbuf_pool);
2417         if (unlikely(err)) {
2418                 if (!allocerr_warned) {
2419                         VHOST_LOG_DATA(ERR,
2420                                 "Failed to copy desc to mbuf on %s.\n",
2421                                 dev->ifname);
2422                         allocerr_warned = true;
2423                 }
2424                 return -1;
2425         }
2426
2427         return 0;
2428 }
2429
2430 static __rte_always_inline int
2431 virtio_dev_tx_single_packed(struct virtio_net *dev,
2432                             struct vhost_virtqueue *vq,
2433                             struct rte_mempool *mbuf_pool,
2434                             struct rte_mbuf *pkts)
2435 {
2436
2437         uint16_t buf_id, desc_count = 0;
2438         int ret;
2439
2440         ret = vhost_dequeue_single_packed(dev, vq, mbuf_pool, pkts, &buf_id,
2441                                         &desc_count);
2442
2443         if (likely(desc_count > 0)) {
2444                 if (virtio_net_is_inorder(dev))
2445                         vhost_shadow_dequeue_single_packed_inorder(vq, buf_id,
2446                                                                    desc_count);
2447                 else
2448                         vhost_shadow_dequeue_single_packed(vq, buf_id,
2449                                         desc_count);
2450
2451                 vq_inc_last_avail_packed(vq, desc_count);
2452         }
2453
2454         return ret;
2455 }
2456
2457 static __rte_noinline uint16_t
2458 virtio_dev_tx_packed(struct virtio_net *dev,
2459                      struct vhost_virtqueue *__rte_restrict vq,
2460                      struct rte_mempool *mbuf_pool,
2461                      struct rte_mbuf **__rte_restrict pkts,
2462                      uint32_t count)
2463 {
2464         uint32_t pkt_idx = 0;
2465
2466         if (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts, count))
2467                 return 0;
2468
2469         do {
2470                 rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
2471
2472                 if (count - pkt_idx >= PACKED_BATCH_SIZE) {
2473                         if (!virtio_dev_tx_batch_packed(dev, vq,
2474                                                         &pkts[pkt_idx])) {
2475                                 pkt_idx += PACKED_BATCH_SIZE;
2476                                 continue;
2477                         }
2478                 }
2479
2480                 if (virtio_dev_tx_single_packed(dev, vq, mbuf_pool,
2481                                                 pkts[pkt_idx]))
2482                         break;
2483                 pkt_idx++;
2484         } while (pkt_idx < count);
2485
2486         if (pkt_idx != count)
2487                 rte_pktmbuf_free_bulk(&pkts[pkt_idx], count - pkt_idx);
2488
2489         if (vq->shadow_used_idx) {
2490                 do_data_copy_dequeue(vq);
2491
2492                 vhost_flush_dequeue_shadow_packed(dev, vq);
2493                 vhost_vring_call_packed(dev, vq);
2494         }
2495
2496         return pkt_idx;
2497 }
2498
2499 uint16_t
2500 rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
2501         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
2502 {
2503         struct virtio_net *dev;
2504         struct rte_mbuf *rarp_mbuf = NULL;
2505         struct vhost_virtqueue *vq;
2506         int16_t success = 1;
2507
2508         dev = get_device(vid);
2509         if (!dev)
2510                 return 0;
2511
2512         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
2513                 VHOST_LOG_DATA(ERR,
2514                         "(%d) %s: built-in vhost net backend is disabled.\n",
2515                         dev->vid, __func__);
2516                 return 0;
2517         }
2518
2519         if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
2520                 VHOST_LOG_DATA(ERR,
2521                         "(%d) %s: invalid virtqueue idx %d.\n",
2522                         dev->vid, __func__, queue_id);
2523                 return 0;
2524         }
2525
2526         vq = dev->virtqueue[queue_id];
2527
2528         if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
2529                 return 0;
2530
2531         if (unlikely(!vq->enabled)) {
2532                 count = 0;
2533                 goto out_access_unlock;
2534         }
2535
2536         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
2537                 vhost_user_iotlb_rd_lock(vq);
2538
2539         if (unlikely(!vq->access_ok))
2540                 if (unlikely(vring_translate(dev, vq) < 0)) {
2541                         count = 0;
2542                         goto out;
2543                 }
2544
2545         /*
2546          * Construct a RARP broadcast packet, and inject it to the "pkts"
2547          * array, to looks like that guest actually send such packet.
2548          *
2549          * Check user_send_rarp() for more information.
2550          *
2551          * broadcast_rarp shares a cacheline in the virtio_net structure
2552          * with some fields that are accessed during enqueue and
2553          * __atomic_compare_exchange_n causes a write if performed compare
2554          * and exchange. This could result in false sharing between enqueue
2555          * and dequeue.
2556          *
2557          * Prevent unnecessary false sharing by reading broadcast_rarp first
2558          * and only performing compare and exchange if the read indicates it
2559          * is likely to be set.
2560          */
2561         if (unlikely(__atomic_load_n(&dev->broadcast_rarp, __ATOMIC_ACQUIRE) &&
2562                         __atomic_compare_exchange_n(&dev->broadcast_rarp,
2563                         &success, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED))) {
2564
2565                 rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
2566                 if (rarp_mbuf == NULL) {
2567                         VHOST_LOG_DATA(ERR, "Failed to make RARP packet.\n");
2568                         count = 0;
2569                         goto out;
2570                 }
2571                 count -= 1;
2572         }
2573
2574         if (vq_is_packed(dev))
2575                 count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count);
2576         else
2577                 count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);
2578
2579 out:
2580         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
2581                 vhost_user_iotlb_rd_unlock(vq);
2582
2583 out_access_unlock:
2584         rte_spinlock_unlock(&vq->access_lock);
2585
2586         if (unlikely(rarp_mbuf != NULL)) {
2587                 /*
2588                  * Inject it to the head of "pkts" array, so that switch's mac
2589                  * learning table will get updated first.
2590                  */
2591                 memmove(&pkts[1], pkts, count * sizeof(struct rte_mbuf *));
2592                 pkts[0] = rarp_mbuf;
2593                 count += 1;
2594         }
2595
2596         return count;
2597 }