lib/librte_vhost/virtio_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdbool.h>
   7 #include <linux/virtio_net.h>
   8
   9 #include <rte_mbuf.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_ether.h>
  12 #include <rte_ip.h>
  13 #include <rte_vhost.h>
  14 #include <rte_tcp.h>
  15 #include <rte_udp.h>
  16 #include <rte_sctp.h>
  17 #include <rte_arp.h>
  18 #include <rte_spinlock.h>
  19 #include <rte_malloc.h>
  20
  21 #include "iotlb.h"
  22 #include "vhost.h"
  23
  24 #define MAX_PKT_BURST 32
  25
  26 #define MAX_BATCH_LEN 256
  27
  28 static  __rte_always_inline bool
  29 rxvq_is_mergeable(struct virtio_net *dev)
  30 {
  31         return dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF);
  32 }
  33
  34 static  __rte_always_inline bool
  35 virtio_net_is_inorder(struct virtio_net *dev)
  36 {
  37         return dev->features & (1ULL << VIRTIO_F_IN_ORDER);
  38 }
  39
  40 static bool
  41 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
  42 {
  43         return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring;
  44 }
  45
  46 static inline void
  47 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
  48 {
  49         struct batch_copy_elem *elem = vq->batch_copy_elems;
  50         uint16_t count = vq->batch_copy_nb_elems;
  51         int i;
  52
  53         for (i = 0; i < count; i++) {
  54                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
  55                 vhost_log_cache_write_iova(dev, vq, elem[i].log_addr,
  56                                            elem[i].len);
  57                 PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
  58         }
  59
  60         vq->batch_copy_nb_elems = 0;
  61 }
  62
  63 static inline void
  64 do_data_copy_dequeue(struct vhost_virtqueue *vq)
  65 {
  66         struct batch_copy_elem *elem = vq->batch_copy_elems;
  67         uint16_t count = vq->batch_copy_nb_elems;
  68         int i;
  69
  70         for (i = 0; i < count; i++)
  71                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
  72
  73         vq->batch_copy_nb_elems = 0;
  74 }
  75
  76 static __rte_always_inline void
  77 do_flush_shadow_used_ring_split(struct virtio_net *dev,
  78                         struct vhost_virtqueue *vq,
  79                         uint16_t to, uint16_t from, uint16_t size)
  80 {
  81         rte_memcpy(&vq->used->ring[to],
  82                         &vq->shadow_used_split[from],
  83                         size * sizeof(struct vring_used_elem));
  84         vhost_log_cache_used_vring(dev, vq,
  85                         offsetof(struct vring_used, ring[to]),
  86                         size * sizeof(struct vring_used_elem));
  87 }
  88
  89 static __rte_always_inline void
  90 flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
  91 {
  92         uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
  93
  94         if (used_idx + vq->shadow_used_idx <= vq->size) {
  95                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0,
  96                                           vq->shadow_used_idx);
  97         } else {
  98                 uint16_t size;
  99
 100                 /* update used ring interval [used_idx, vq->size] */
 101                 size = vq->size - used_idx;
 102                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size);
 103
 104                 /* update the left half used ring interval [0, left_size] */
 105                 do_flush_shadow_used_ring_split(dev, vq, 0, size,
 106                                           vq->shadow_used_idx - size);
 107         }
 108         vq->last_used_idx += vq->shadow_used_idx;
 109
 110         vhost_log_cache_sync(dev, vq);
 111
 112         __atomic_add_fetch(&vq->used->idx, vq->shadow_used_idx,
 113                            __ATOMIC_RELEASE);
 114         vq->shadow_used_idx = 0;
 115         vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
 116                 sizeof(vq->used->idx));
 117 }
 118
 119 static __rte_always_inline void
 120 update_shadow_used_ring_split(struct vhost_virtqueue *vq,
 121                          uint16_t desc_idx, uint32_t len)
 122 {
 123         uint16_t i = vq->shadow_used_idx++;
 124
 125         vq->shadow_used_split[i].id  = desc_idx;
 126         vq->shadow_used_split[i].len = len;
 127 }
 128
 129 static __rte_always_inline void
 130 vhost_flush_enqueue_shadow_packed(struct virtio_net *dev,
 131                                   struct vhost_virtqueue *vq)
 132 {
 133         int i;
 134         uint16_t used_idx = vq->last_used_idx;
 135         uint16_t head_idx = vq->last_used_idx;
 136         uint16_t head_flags = 0;
 137
 138         /* Split loop in two to save memory barriers */
 139         for (i = 0; i < vq->shadow_used_idx; i++) {
 140                 vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id;
 141                 vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len;
 142
 143                 used_idx += vq->shadow_used_packed[i].count;
 144                 if (used_idx >= vq->size)
 145                         used_idx -= vq->size;
 146         }
 147
 148         rte_smp_wmb();
 149
 150         for (i = 0; i < vq->shadow_used_idx; i++) {
 151                 uint16_t flags;
 152
 153                 if (vq->shadow_used_packed[i].len)
 154                         flags = VRING_DESC_F_WRITE;
 155                 else
 156                         flags = 0;
 157
 158                 if (vq->used_wrap_counter) {
 159                         flags |= VRING_DESC_F_USED;
 160                         flags |= VRING_DESC_F_AVAIL;
 161                 } else {
 162                         flags &= ~VRING_DESC_F_USED;
 163                         flags &= ~VRING_DESC_F_AVAIL;
 164                 }
 165
 166                 if (i > 0) {
 167                         vq->desc_packed[vq->last_used_idx].flags = flags;
 168
 169                         vhost_log_cache_used_vring(dev, vq,
 170                                         vq->last_used_idx *
 171                                         sizeof(struct vring_packed_desc),
 172                                         sizeof(struct vring_packed_desc));
 173                 } else {
 174                         head_idx = vq->last_used_idx;
 175                         head_flags = flags;
 176                 }
 177
 178                 vq_inc_last_used_packed(vq, vq->shadow_used_packed[i].count);
 179         }
 180
 181         vq->desc_packed[head_idx].flags = head_flags;
 182
 183         vhost_log_cache_used_vring(dev, vq,
 184                                 head_idx *
 185                                 sizeof(struct vring_packed_desc),
 186                                 sizeof(struct vring_packed_desc));
 187
 188         vq->shadow_used_idx = 0;
 189         vhost_log_cache_sync(dev, vq);
 190 }
 191
 192 static __rte_always_inline void
 193 vhost_flush_dequeue_shadow_packed(struct virtio_net *dev,
 194                                   struct vhost_virtqueue *vq)
 195 {
 196         struct vring_used_elem_packed *used_elem = &vq->shadow_used_packed[0];
 197
 198         vq->desc_packed[vq->shadow_last_used_idx].id = used_elem->id;
 199         rte_smp_wmb();
 200         vq->desc_packed[vq->shadow_last_used_idx].flags = used_elem->flags;
 201
 202         vhost_log_cache_used_vring(dev, vq, vq->shadow_last_used_idx *
 203                                    sizeof(struct vring_packed_desc),
 204                                    sizeof(struct vring_packed_desc));
 205         vq->shadow_used_idx = 0;
 206         vhost_log_cache_sync(dev, vq);
 207 }
 208
 209 static __rte_always_inline void
 210 vhost_flush_enqueue_batch_packed(struct virtio_net *dev,
 211                                  struct vhost_virtqueue *vq,
 212                                  uint64_t *lens,
 213                                  uint16_t *ids)
 214 {
 215         uint16_t i;
 216         uint16_t flags;
 217
 218         if (vq->shadow_used_idx) {
 219                 do_data_copy_enqueue(dev, vq);
 220                 vhost_flush_enqueue_shadow_packed(dev, vq);
 221         }
 222
 223         flags = PACKED_DESC_ENQUEUE_USED_FLAG(vq->used_wrap_counter);
 224
 225         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
 226                 vq->desc_packed[vq->last_used_idx + i].id = ids[i];
 227                 vq->desc_packed[vq->last_used_idx + i].len = lens[i];
 228         }
 229
 230         rte_smp_wmb();
 231
 232         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
 233                 vq->desc_packed[vq->last_used_idx + i].flags = flags;
 234
 235         vhost_log_cache_used_vring(dev, vq, vq->last_used_idx *
 236                                    sizeof(struct vring_packed_desc),
 237                                    sizeof(struct vring_packed_desc) *
 238                                    PACKED_BATCH_SIZE);
 239         vhost_log_cache_sync(dev, vq);
 240
 241         vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);
 242 }
 243
 244 static __rte_always_inline void
 245 vhost_shadow_dequeue_batch_packed_inorder(struct vhost_virtqueue *vq,
 246                                           uint16_t id)
 247 {
 248         vq->shadow_used_packed[0].id = id;
 249
 250         if (!vq->shadow_used_idx) {
 251                 vq->shadow_last_used_idx = vq->last_used_idx;
 252                 vq->shadow_used_packed[0].flags =
 253                         PACKED_DESC_DEQUEUE_USED_FLAG(vq->used_wrap_counter);
 254                 vq->shadow_used_packed[0].len = 0;
 255                 vq->shadow_used_packed[0].count = 1;
 256                 vq->shadow_used_idx++;
 257         }
 258
 259         vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);
 260 }
 261
 262 static __rte_always_inline void
 263 vhost_shadow_dequeue_batch_packed(struct virtio_net *dev,
 264                                   struct vhost_virtqueue *vq,
 265                                   uint16_t *ids)
 266 {
 267         uint16_t flags;
 268         uint16_t i;
 269         uint16_t begin;
 270
 271         flags = PACKED_DESC_DEQUEUE_USED_FLAG(vq->used_wrap_counter);
 272
 273         if (!vq->shadow_used_idx) {
 274                 vq->shadow_last_used_idx = vq->last_used_idx;
 275                 vq->shadow_used_packed[0].id  = ids[0];
 276                 vq->shadow_used_packed[0].len = 0;
 277                 vq->shadow_used_packed[0].count = 1;
 278                 vq->shadow_used_packed[0].flags = flags;
 279                 vq->shadow_used_idx++;
 280                 begin = 1;
 281         } else
 282                 begin = 0;
 283
 284         vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE) {
 285                 vq->desc_packed[vq->last_used_idx + i].id = ids[i];
 286                 vq->desc_packed[vq->last_used_idx + i].len = 0;
 287         }
 288
 289         rte_smp_wmb();
 290         vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE)
 291                 vq->desc_packed[vq->last_used_idx + i].flags = flags;
 292
 293         vhost_log_cache_used_vring(dev, vq, vq->last_used_idx *
 294                                    sizeof(struct vring_packed_desc),
 295                                    sizeof(struct vring_packed_desc) *
 296                                    PACKED_BATCH_SIZE);
 297         vhost_log_cache_sync(dev, vq);
 298
 299         vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);
 300 }
 301
 302 static __rte_always_inline void
 303 vhost_shadow_dequeue_single_packed(struct vhost_virtqueue *vq,
 304                                    uint16_t buf_id,
 305                                    uint16_t count)
 306 {
 307         uint16_t flags;
 308
 309         flags = vq->desc_packed[vq->last_used_idx].flags;
 310         if (vq->used_wrap_counter) {
 311                 flags |= VRING_DESC_F_USED;
 312                 flags |= VRING_DESC_F_AVAIL;
 313         } else {
 314                 flags &= ~VRING_DESC_F_USED;
 315                 flags &= ~VRING_DESC_F_AVAIL;
 316         }
 317
 318         if (!vq->shadow_used_idx) {
 319                 vq->shadow_last_used_idx = vq->last_used_idx;
 320
 321                 vq->shadow_used_packed[0].id  = buf_id;
 322                 vq->shadow_used_packed[0].len = 0;
 323                 vq->shadow_used_packed[0].flags = flags;
 324                 vq->shadow_used_idx++;
 325         } else {
 326                 vq->desc_packed[vq->last_used_idx].id = buf_id;
 327                 vq->desc_packed[vq->last_used_idx].len = 0;
 328                 vq->desc_packed[vq->last_used_idx].flags = flags;
 329         }
 330
 331         vq_inc_last_used_packed(vq, count);
 332 }
 333
 334 static __rte_always_inline void
 335 vhost_shadow_dequeue_single_packed_inorder(struct vhost_virtqueue *vq,
 336                                            uint16_t buf_id,
 337                                            uint16_t count)
 338 {
 339         uint16_t flags;
 340
 341         vq->shadow_used_packed[0].id = buf_id;
 342
 343         flags = vq->desc_packed[vq->last_used_idx].flags;
 344         if (vq->used_wrap_counter) {
 345                 flags |= VRING_DESC_F_USED;
 346                 flags |= VRING_DESC_F_AVAIL;
 347         } else {
 348                 flags &= ~VRING_DESC_F_USED;
 349                 flags &= ~VRING_DESC_F_AVAIL;
 350         }
 351
 352         if (!vq->shadow_used_idx) {
 353                 vq->shadow_last_used_idx = vq->last_used_idx;
 354                 vq->shadow_used_packed[0].len = 0;
 355                 vq->shadow_used_packed[0].flags = flags;
 356                 vq->shadow_used_idx++;
 357         }
 358
 359         vq_inc_last_used_packed(vq, count);
 360 }
 361
 362 static __rte_always_inline void
 363 vhost_shadow_enqueue_single_packed(struct virtio_net *dev,
 364                                    struct vhost_virtqueue *vq,
 365                                    uint32_t len[],
 366                                    uint16_t id[],
 367                                    uint16_t count[],
 368                                    uint16_t num_buffers)
 369 {
 370         uint16_t i;
 371         for (i = 0; i < num_buffers; i++) {
 372                 /* enqueue shadow flush action aligned with batch num */
 373                 if (!vq->shadow_used_idx)
 374                         vq->shadow_aligned_idx = vq->last_used_idx &
 375                                 PACKED_BATCH_MASK;
 376                 vq->shadow_used_packed[vq->shadow_used_idx].id  = id[i];
 377                 vq->shadow_used_packed[vq->shadow_used_idx].len = len[i];
 378                 vq->shadow_used_packed[vq->shadow_used_idx].count = count[i];
 379                 vq->shadow_aligned_idx += count[i];
 380                 vq->shadow_used_idx++;
 381         }
 382
 383         if (vq->shadow_aligned_idx >= PACKED_BATCH_SIZE) {
 384                 do_data_copy_enqueue(dev, vq);
 385                 vhost_flush_enqueue_shadow_packed(dev, vq);
 386         }
 387 }
 388
 389 /* avoid write operation when necessary, to lessen cache issues */
 390 #define ASSIGN_UNLESS_EQUAL(var, val) do {      \
 391         if ((var) != (val))                     \
 392                 (var) = (val);                  \
 393 } while (0)
 394
 395 static __rte_always_inline void
 396 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 397 {
 398         uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK;
 399
 400         if (m_buf->ol_flags & PKT_TX_TCP_SEG)
 401                 csum_l4 |= PKT_TX_TCP_CKSUM;
 402
 403         if (csum_l4) {
 404                 net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
 405                 net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len;
 406
 407                 switch (csum_l4) {
 408                 case PKT_TX_TCP_CKSUM:
 409                         net_hdr->csum_offset = (offsetof(struct rte_tcp_hdr,
 410                                                 cksum));
 411                         break;
 412                 case PKT_TX_UDP_CKSUM:
 413                         net_hdr->csum_offset = (offsetof(struct rte_udp_hdr,
 414                                                 dgram_cksum));
 415                         break;
 416                 case PKT_TX_SCTP_CKSUM:
 417                         net_hdr->csum_offset = (offsetof(struct rte_sctp_hdr,
 418                                                 cksum));
 419                         break;
 420                 }
 421         } else {
 422                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0);
 423                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0);
 424                 ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0);
 425         }
 426
 427         /* IP cksum verification cannot be bypassed, then calculate here */
 428         if (m_buf->ol_flags & PKT_TX_IP_CKSUM) {
 429                 struct rte_ipv4_hdr *ipv4_hdr;
 430
 431                 ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct rte_ipv4_hdr *,
 432                                                    m_buf->l2_len);
 433                 ipv4_hdr->hdr_checksum = 0;
 434                 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
 435         }
 436
 437         if (m_buf->ol_flags & PKT_TX_TCP_SEG) {
 438                 if (m_buf->ol_flags & PKT_TX_IPV4)
 439                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
 440                 else
 441                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
 442                 net_hdr->gso_size = m_buf->tso_segsz;
 443                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len
 444                                         + m_buf->l4_len;
 445         } else if (m_buf->ol_flags & PKT_TX_UDP_SEG) {
 446                 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
 447                 net_hdr->gso_size = m_buf->tso_segsz;
 448                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len +
 449                         m_buf->l4_len;
 450         } else {
 451                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0);
 452                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0);
 453                 ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0);
 454         }
 455 }
 456
 457 static __rte_always_inline int
 458 map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 459                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 460                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 461 {
 462         uint16_t vec_id = *vec_idx;
 463
 464         while (desc_len) {
 465                 uint64_t desc_addr;
 466                 uint64_t desc_chunck_len = desc_len;
 467
 468                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 469                         return -1;
 470
 471                 desc_addr = vhost_iova_to_vva(dev, vq,
 472                                 desc_iova,
 473                                 &desc_chunck_len,
 474                                 perm);
 475                 if (unlikely(!desc_addr))
 476                         return -1;
 477
 478                 rte_prefetch0((void *)(uintptr_t)desc_addr);
 479
 480                 buf_vec[vec_id].buf_iova = desc_iova;
 481                 buf_vec[vec_id].buf_addr = desc_addr;
 482                 buf_vec[vec_id].buf_len  = desc_chunck_len;
 483
 484                 desc_len -= desc_chunck_len;
 485                 desc_iova += desc_chunck_len;
 486                 vec_id++;
 487         }
 488         *vec_idx = vec_id;
 489
 490         return 0;
 491 }
 492
 493 static __rte_always_inline int
 494 fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 495                          uint32_t avail_idx, uint16_t *vec_idx,
 496                          struct buf_vector *buf_vec, uint16_t *desc_chain_head,
 497                          uint32_t *desc_chain_len, uint8_t perm)
 498 {
 499         uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
 500         uint16_t vec_id = *vec_idx;
 501         uint32_t len    = 0;
 502         uint64_t dlen;
 503         uint32_t nr_descs = vq->size;
 504         uint32_t cnt    = 0;
 505         struct vring_desc *descs = vq->desc;
 506         struct vring_desc *idesc = NULL;
 507
 508         if (unlikely(idx >= vq->size))
 509                 return -1;
 510
 511         *desc_chain_head = idx;
 512
 513         if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
 514                 dlen = vq->desc[idx].len;
 515                 nr_descs = dlen / sizeof(struct vring_desc);
 516                 if (unlikely(nr_descs > vq->size))
 517                         return -1;
 518
 519                 descs = (struct vring_desc *)(uintptr_t)
 520                         vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
 521                                                 &dlen,
 522                                                 VHOST_ACCESS_RO);
 523                 if (unlikely(!descs))
 524                         return -1;
 525
 526                 if (unlikely(dlen < vq->desc[idx].len)) {
 527                         /*
 528                          * The indirect desc table is not contiguous
 529                          * in process VA space, we have to copy it.
 530                          */
 531                         idesc = vhost_alloc_copy_ind_table(dev, vq,
 532                                         vq->desc[idx].addr, vq->desc[idx].len);
 533                         if (unlikely(!idesc))
 534                                 return -1;
 535
 536                         descs = idesc;
 537                 }
 538
 539                 idx = 0;
 540         }
 541
 542         while (1) {
 543                 if (unlikely(idx >= nr_descs || cnt++ >= nr_descs)) {
 544                         free_ind_table(idesc);
 545                         return -1;
 546                 }
 547
 548                 len += descs[idx].len;
 549
 550                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 551                                                 descs[idx].addr, descs[idx].len,
 552                                                 perm))) {
 553                         free_ind_table(idesc);
 554                         return -1;
 555                 }
 556
 557                 if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
 558                         break;
 559
 560                 idx = descs[idx].next;
 561         }
 562
 563         *desc_chain_len = len;
 564         *vec_idx = vec_id;
 565
 566         if (unlikely(!!idesc))
 567                 free_ind_table(idesc);
 568
 569         return 0;
 570 }
 571
 572 /*
 573  * Returns -1 on fail, 0 on success
 574  */
 575 static inline int
 576 reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 577                                 uint32_t size, struct buf_vector *buf_vec,
 578                                 uint16_t *num_buffers, uint16_t avail_head,
 579                                 uint16_t *nr_vec)
 580 {
 581         uint16_t cur_idx;
 582         uint16_t vec_idx = 0;
 583         uint16_t max_tries, tries = 0;
 584
 585         uint16_t head_idx = 0;
 586         uint32_t len = 0;
 587
 588         *num_buffers = 0;
 589         cur_idx  = vq->last_avail_idx;
 590
 591         if (rxvq_is_mergeable(dev))
 592                 max_tries = vq->size - 1;
 593         else
 594                 max_tries = 1;
 595
 596         while (size > 0) {
 597                 if (unlikely(cur_idx == avail_head))
 598                         return -1;
 599                 /*
 600                  * if we tried all available ring items, and still
 601                  * can't get enough buf, it means something abnormal
 602                  * happened.
 603                  */
 604                 if (unlikely(++tries > max_tries))
 605                         return -1;
 606
 607                 if (unlikely(fill_vec_buf_split(dev, vq, cur_idx,
 608                                                 &vec_idx, buf_vec,
 609                                                 &head_idx, &len,
 610                                                 VHOST_ACCESS_RW) < 0))
 611                         return -1;
 612                 len = RTE_MIN(len, size);
 613                 update_shadow_used_ring_split(vq, head_idx, len);
 614                 size -= len;
 615
 616                 cur_idx++;
 617                 *num_buffers += 1;
 618         }
 619
 620         *nr_vec = vec_idx;
 621
 622         return 0;
 623 }
 624
 625 static __rte_always_inline int
 626 fill_vec_buf_packed_indirect(struct virtio_net *dev,
 627                         struct vhost_virtqueue *vq,
 628                         struct vring_packed_desc *desc, uint16_t *vec_idx,
 629                         struct buf_vector *buf_vec, uint32_t *len, uint8_t perm)
 630 {
 631         uint16_t i;
 632         uint32_t nr_descs;
 633         uint16_t vec_id = *vec_idx;
 634         uint64_t dlen;
 635         struct vring_packed_desc *descs, *idescs = NULL;
 636
 637         dlen = desc->len;
 638         descs = (struct vring_packed_desc *)(uintptr_t)
 639                 vhost_iova_to_vva(dev, vq, desc->addr, &dlen, VHOST_ACCESS_RO);
 640         if (unlikely(!descs))
 641                 return -1;
 642
 643         if (unlikely(dlen < desc->len)) {
 644                 /*
 645                  * The indirect desc table is not contiguous
 646                  * in process VA space, we have to copy it.
 647                  */
 648                 idescs = vhost_alloc_copy_ind_table(dev,
 649                                 vq, desc->addr, desc->len);
 650                 if (unlikely(!idescs))
 651                         return -1;
 652
 653                 descs = idescs;
 654         }
 655
 656         nr_descs =  desc->len / sizeof(struct vring_packed_desc);
 657         if (unlikely(nr_descs >= vq->size)) {
 658                 free_ind_table(idescs);
 659                 return -1;
 660         }
 661
 662         for (i = 0; i < nr_descs; i++) {
 663                 if (unlikely(vec_id >= BUF_VECTOR_MAX)) {
 664                         free_ind_table(idescs);
 665                         return -1;
 666                 }
 667
 668                 *len += descs[i].len;
 669                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 670                                                 descs[i].addr, descs[i].len,
 671                                                 perm)))
 672                         return -1;
 673         }
 674         *vec_idx = vec_id;
 675
 676         if (unlikely(!!idescs))
 677                 free_ind_table(idescs);
 678
 679         return 0;
 680 }
 681
 682 static __rte_always_inline int
 683 fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 684                                 uint16_t avail_idx, uint16_t *desc_count,
 685                                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 686                                 uint16_t *buf_id, uint32_t *len, uint8_t perm)
 687 {
 688         bool wrap_counter = vq->avail_wrap_counter;
 689         struct vring_packed_desc *descs = vq->desc_packed;
 690         uint16_t vec_id = *vec_idx;
 691
 692         if (avail_idx < vq->last_avail_idx)
 693                 wrap_counter ^= 1;
 694
 695         /*
 696          * Perform a load-acquire barrier in desc_is_avail to
 697          * enforce the ordering between desc flags and desc
 698          * content.
 699          */
 700         if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
 701                 return -1;
 702
 703         *desc_count = 0;
 704         *len = 0;
 705
 706         while (1) {
 707                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 708                         return -1;
 709
 710                 if (unlikely(*desc_count >= vq->size))
 711                         return -1;
 712
 713                 *desc_count += 1;
 714                 *buf_id = descs[avail_idx].id;
 715
 716                 if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) {
 717                         if (unlikely(fill_vec_buf_packed_indirect(dev, vq,
 718                                                         &descs[avail_idx],
 719                                                         &vec_id, buf_vec,
 720                                                         len, perm) < 0))
 721                                 return -1;
 722                 } else {
 723                         *len += descs[avail_idx].len;
 724
 725                         if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 726                                                         descs[avail_idx].addr,
 727                                                         descs[avail_idx].len,
 728                                                         perm)))
 729                                 return -1;
 730                 }
 731
 732                 if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0)
 733                         break;
 734
 735                 if (++avail_idx >= vq->size) {
 736                         avail_idx -= vq->size;
 737                         wrap_counter ^= 1;
 738                 }
 739         }
 740
 741         *vec_idx = vec_id;
 742
 743         return 0;
 744 }
 745
 746 static __rte_noinline void
 747 copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 748                 struct buf_vector *buf_vec,
 749                 struct virtio_net_hdr_mrg_rxbuf *hdr)
 750 {
 751         uint64_t len;
 752         uint64_t remain = dev->vhost_hlen;
 753         uint64_t src = (uint64_t)(uintptr_t)hdr, dst;
 754         uint64_t iova = buf_vec->buf_iova;
 755
 756         while (remain) {
 757                 len = RTE_MIN(remain,
 758                                 buf_vec->buf_len);
 759                 dst = buf_vec->buf_addr;
 760                 rte_memcpy((void *)(uintptr_t)dst,
 761                                 (void *)(uintptr_t)src,
 762                                 len);
 763
 764                 PRINT_PACKET(dev, (uintptr_t)dst,
 765                                 (uint32_t)len, 0);
 766                 vhost_log_cache_write_iova(dev, vq,
 767                                 iova, len);
 768
 769                 remain -= len;
 770                 iova += len;
 771                 src += len;
 772                 buf_vec++;
 773         }
 774 }
 775
 776 static __rte_always_inline int
 777 copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 778                             struct rte_mbuf *m, struct buf_vector *buf_vec,
 779                             uint16_t nr_vec, uint16_t num_buffers)
 780 {
 781         uint32_t vec_idx = 0;
 782         uint32_t mbuf_offset, mbuf_avail;
 783         uint32_t buf_offset, buf_avail;
 784         uint64_t buf_addr, buf_iova, buf_len;
 785         uint32_t cpy_len;
 786         uint64_t hdr_addr;
 787         struct rte_mbuf *hdr_mbuf;
 788         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 789         struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
 790         int error = 0;
 791
 792         if (unlikely(m == NULL)) {
 793                 error = -1;
 794                 goto out;
 795         }
 796
 797         buf_addr = buf_vec[vec_idx].buf_addr;
 798         buf_iova = buf_vec[vec_idx].buf_iova;
 799         buf_len = buf_vec[vec_idx].buf_len;
 800
 801         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 802                 error = -1;
 803                 goto out;
 804         }
 805
 806         hdr_mbuf = m;
 807         hdr_addr = buf_addr;
 808         if (unlikely(buf_len < dev->vhost_hlen))
 809                 hdr = &tmp_hdr;
 810         else
 811                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
 812
 813         VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n",
 814                 dev->vid, num_buffers);
 815
 816         if (unlikely(buf_len < dev->vhost_hlen)) {
 817                 buf_offset = dev->vhost_hlen - buf_len;
 818                 vec_idx++;
 819                 buf_addr = buf_vec[vec_idx].buf_addr;
 820                 buf_iova = buf_vec[vec_idx].buf_iova;
 821                 buf_len = buf_vec[vec_idx].buf_len;
 822                 buf_avail = buf_len - buf_offset;
 823         } else {
 824                 buf_offset = dev->vhost_hlen;
 825                 buf_avail = buf_len - dev->vhost_hlen;
 826         }
 827
 828         mbuf_avail  = rte_pktmbuf_data_len(m);
 829         mbuf_offset = 0;
 830         while (mbuf_avail != 0 || m->next != NULL) {
 831                 /* done with current buf, get the next one */
 832                 if (buf_avail == 0) {
 833                         vec_idx++;
 834                         if (unlikely(vec_idx >= nr_vec)) {
 835                                 error = -1;
 836                                 goto out;
 837                         }
 838
 839                         buf_addr = buf_vec[vec_idx].buf_addr;
 840                         buf_iova = buf_vec[vec_idx].buf_iova;
 841                         buf_len = buf_vec[vec_idx].buf_len;
 842
 843                         buf_offset = 0;
 844                         buf_avail  = buf_len;
 845                 }
 846
 847                 /* done with current mbuf, get the next one */
 848                 if (mbuf_avail == 0) {
 849                         m = m->next;
 850
 851                         mbuf_offset = 0;
 852                         mbuf_avail  = rte_pktmbuf_data_len(m);
 853                 }
 854
 855                 if (hdr_addr) {
 856                         virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
 857                         if (rxvq_is_mergeable(dev))
 858                                 ASSIGN_UNLESS_EQUAL(hdr->num_buffers,
 859                                                 num_buffers);
 860
 861                         if (unlikely(hdr == &tmp_hdr)) {
 862                                 copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr);
 863                         } else {
 864                                 PRINT_PACKET(dev, (uintptr_t)hdr_addr,
 865                                                 dev->vhost_hlen, 0);
 866                                 vhost_log_cache_write_iova(dev, vq,
 867                                                 buf_vec[0].buf_iova,
 868                                                 dev->vhost_hlen);
 869                         }
 870
 871                         hdr_addr = 0;
 872                 }
 873
 874                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 875
 876                 if (likely(cpy_len > MAX_BATCH_LEN ||
 877                                         vq->batch_copy_nb_elems >= vq->size)) {
 878                         rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),
 879                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 880                                 cpy_len);
 881                         vhost_log_cache_write_iova(dev, vq,
 882                                                    buf_iova + buf_offset,
 883                                                    cpy_len);
 884                         PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),
 885                                 cpy_len, 0);
 886                 } else {
 887                         batch_copy[vq->batch_copy_nb_elems].dst =
 888                                 (void *)((uintptr_t)(buf_addr + buf_offset));
 889                         batch_copy[vq->batch_copy_nb_elems].src =
 890                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
 891                         batch_copy[vq->batch_copy_nb_elems].log_addr =
 892                                 buf_iova + buf_offset;
 893                         batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 894                         vq->batch_copy_nb_elems++;
 895                 }
 896
 897                 mbuf_avail  -= cpy_len;
 898                 mbuf_offset += cpy_len;
 899                 buf_avail  -= cpy_len;
 900                 buf_offset += cpy_len;
 901         }
 902
 903 out:
 904
 905         return error;
 906 }
 907
 908 static __rte_always_inline int
 909 vhost_enqueue_single_packed(struct virtio_net *dev,
 910                             struct vhost_virtqueue *vq,
 911                             struct rte_mbuf *pkt,
 912                             struct buf_vector *buf_vec,
 913                             uint16_t *nr_descs)
 914 {
 915         uint16_t nr_vec = 0;
 916         uint16_t avail_idx = vq->last_avail_idx;
 917         uint16_t max_tries, tries = 0;
 918         uint16_t buf_id = 0;
 919         uint32_t len = 0;
 920         uint16_t desc_count;
 921         uint32_t size = pkt->pkt_len + dev->vhost_hlen;
 922         uint16_t num_buffers = 0;
 923         uint32_t buffer_len[vq->size];
 924         uint16_t buffer_buf_id[vq->size];
 925         uint16_t buffer_desc_count[vq->size];
 926
 927         if (rxvq_is_mergeable(dev))
 928                 max_tries = vq->size - 1;
 929         else
 930                 max_tries = 1;
 931
 932         while (size > 0) {
 933                 /*
 934                  * if we tried all available ring items, and still
 935                  * can't get enough buf, it means something abnormal
 936                  * happened.
 937                  */
 938                 if (unlikely(++tries > max_tries))
 939                         return -1;
 940
 941                 if (unlikely(fill_vec_buf_packed(dev, vq,
 942                                                 avail_idx, &desc_count,
 943                                                 buf_vec, &nr_vec,
 944                                                 &buf_id, &len,
 945                                                 VHOST_ACCESS_RW) < 0))
 946                         return -1;
 947
 948                 len = RTE_MIN(len, size);
 949                 size -= len;
 950
 951                 buffer_len[num_buffers] = len;
 952                 buffer_buf_id[num_buffers] = buf_id;
 953                 buffer_desc_count[num_buffers] = desc_count;
 954                 num_buffers += 1;
 955
 956                 *nr_descs += desc_count;
 957                 avail_idx += desc_count;
 958                 if (avail_idx >= vq->size)
 959                         avail_idx -= vq->size;
 960         }
 961
 962         if (copy_mbuf_to_desc(dev, vq, pkt, buf_vec, nr_vec, num_buffers) < 0)
 963                 return -1;
 964
 965         vhost_shadow_enqueue_single_packed(dev, vq, buffer_len, buffer_buf_id,
 966                                            buffer_desc_count, num_buffers);
 967
 968         return 0;
 969 }
 970
 971 static __rte_noinline uint32_t
 972 virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 973         struct rte_mbuf **pkts, uint32_t count)
 974 {
 975         uint32_t pkt_idx = 0;
 976         uint16_t num_buffers;
 977         struct buf_vector buf_vec[BUF_VECTOR_MAX];
 978         uint16_t avail_head;
 979
 980         /*
 981          * The ordering between avail index and
 982          * desc reads needs to be enforced.
 983          */
 984         avail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE);
 985
 986         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
 987
 988         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 989                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
 990                 uint16_t nr_vec = 0;
 991
 992                 if (unlikely(reserve_avail_buf_split(dev, vq,
 993                                                 pkt_len, buf_vec, &num_buffers,
 994                                                 avail_head, &nr_vec) < 0)) {
 995                         VHOST_LOG_DATA(DEBUG,
 996                                 "(%d) failed to get enough desc from vring\n",
 997                                 dev->vid);
 998                         vq->shadow_used_idx -= num_buffers;
 999                         break;
1000                 }
1001
1002                 VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n",
1003                         dev->vid, vq->last_avail_idx,
1004                         vq->last_avail_idx + num_buffers);
1005
1006                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
1007                                                 buf_vec, nr_vec,
1008                                                 num_buffers) < 0) {
1009                         vq->shadow_used_idx -= num_buffers;
1010                         break;
1011                 }
1012
1013                 vq->last_avail_idx += num_buffers;
1014         }
1015
1016         do_data_copy_enqueue(dev, vq);
1017
1018         if (likely(vq->shadow_used_idx)) {
1019                 flush_shadow_used_ring_split(dev, vq);
1020                 vhost_vring_call_split(dev, vq);
1021         }
1022
1023         return pkt_idx;
1024 }
1025
1026 static __rte_always_inline int
1027 virtio_dev_rx_batch_packed(struct virtio_net *dev,
1028                            struct vhost_virtqueue *vq,
1029                            struct rte_mbuf **pkts)
1030 {
1031         bool wrap_counter = vq->avail_wrap_counter;
1032         struct vring_packed_desc *descs = vq->desc_packed;
1033         uint16_t avail_idx = vq->last_avail_idx;
1034         uint64_t desc_addrs[PACKED_BATCH_SIZE];
1035         struct virtio_net_hdr_mrg_rxbuf *hdrs[PACKED_BATCH_SIZE];
1036         uint32_t buf_offset = dev->vhost_hlen;
1037         uint64_t lens[PACKED_BATCH_SIZE];
1038         uint16_t ids[PACKED_BATCH_SIZE];
1039         uint16_t i;
1040
1041         if (unlikely(avail_idx & PACKED_BATCH_MASK))
1042                 return -1;
1043
1044         if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))
1045                 return -1;
1046
1047         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1048                 if (unlikely(pkts[i]->next != NULL))
1049                         return -1;
1050                 if (unlikely(!desc_is_avail(&descs[avail_idx + i],
1051                                             wrap_counter)))
1052                         return -1;
1053         }
1054
1055         rte_smp_rmb();
1056
1057         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1058                 lens[i] = descs[avail_idx + i].len;
1059
1060         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1061                 if (unlikely(pkts[i]->pkt_len > (lens[i] - buf_offset)))
1062                         return -1;
1063         }
1064
1065         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1066                 desc_addrs[i] = vhost_iova_to_vva(dev, vq,
1067                                                   descs[avail_idx + i].addr,
1068                                                   &lens[i],
1069                                                   VHOST_ACCESS_RW);
1070
1071         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1072                 if (unlikely(lens[i] != descs[avail_idx + i].len))
1073                         return -1;
1074         }
1075
1076         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1077                 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]);
1078                 hdrs[i] = (struct virtio_net_hdr_mrg_rxbuf *)
1079                                         (uintptr_t)desc_addrs[i];
1080                 lens[i] = pkts[i]->pkt_len + dev->vhost_hlen;
1081         }
1082
1083         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1084                 virtio_enqueue_offload(pkts[i], &hdrs[i]->hdr);
1085
1086         vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
1087
1088         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1089                 rte_memcpy((void *)(uintptr_t)(desc_addrs[i] + buf_offset),
1090                            rte_pktmbuf_mtod_offset(pkts[i], void *, 0),
1091                            pkts[i]->pkt_len);
1092         }
1093
1094         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1095                 vhost_log_cache_write_iova(dev, vq, descs[avail_idx + i].addr,
1096                                            lens[i]);
1097
1098         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1099                 ids[i] = descs[avail_idx + i].id;
1100
1101         vhost_flush_enqueue_batch_packed(dev, vq, lens, ids);
1102
1103         return 0;
1104 }
1105
1106 static __rte_always_inline int16_t
1107 virtio_dev_rx_single_packed(struct virtio_net *dev,
1108                             struct vhost_virtqueue *vq,
1109                             struct rte_mbuf *pkt)
1110 {
1111         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1112         uint16_t nr_descs = 0;
1113
1114         rte_smp_rmb();
1115         if (unlikely(vhost_enqueue_single_packed(dev, vq, pkt, buf_vec,
1116                                                  &nr_descs) < 0)) {
1117                 VHOST_LOG_DATA(DEBUG,
1118                                 "(%d) failed to get enough desc from vring\n",
1119                                 dev->vid);
1120                 return -1;
1121         }
1122
1123         VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n",
1124                         dev->vid, vq->last_avail_idx,
1125                         vq->last_avail_idx + nr_descs);
1126
1127         vq_inc_last_avail_packed(vq, nr_descs);
1128
1129         return 0;
1130 }
1131
1132 static __rte_noinline uint32_t
1133 virtio_dev_rx_packed(struct virtio_net *dev,
1134                      struct vhost_virtqueue *vq,
1135                      struct rte_mbuf **pkts,
1136                      uint32_t count)
1137 {
1138         uint32_t pkt_idx = 0;
1139         uint32_t remained = count;
1140
1141         do {
1142                 rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
1143
1144                 if (remained >= PACKED_BATCH_SIZE) {
1145                         if (!virtio_dev_rx_batch_packed(dev, vq,
1146                                                         &pkts[pkt_idx])) {
1147                                 pkt_idx += PACKED_BATCH_SIZE;
1148                                 remained -= PACKED_BATCH_SIZE;
1149                                 continue;
1150                         }
1151                 }
1152
1153                 if (virtio_dev_rx_single_packed(dev, vq, pkts[pkt_idx]))
1154                         break;
1155                 pkt_idx++;
1156                 remained--;
1157
1158         } while (pkt_idx < count);
1159
1160         if (vq->shadow_used_idx) {
1161                 do_data_copy_enqueue(dev, vq);
1162                 vhost_flush_enqueue_shadow_packed(dev, vq);
1163         }
1164
1165         if (pkt_idx)
1166                 vhost_vring_call_packed(dev, vq);
1167
1168         return pkt_idx;
1169 }
1170
1171 static __rte_always_inline uint32_t
1172 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
1173         struct rte_mbuf **pkts, uint32_t count)
1174 {
1175         struct vhost_virtqueue *vq;
1176         uint32_t nb_tx = 0;
1177
1178         VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
1179         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
1180                 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
1181                         dev->vid, __func__, queue_id);
1182                 return 0;
1183         }
1184
1185         vq = dev->virtqueue[queue_id];
1186
1187         rte_spinlock_lock(&vq->access_lock);
1188
1189         if (unlikely(vq->enabled == 0))
1190                 goto out_access_unlock;
1191
1192         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1193                 vhost_user_iotlb_rd_lock(vq);
1194
1195         if (unlikely(vq->access_ok == 0))
1196                 if (unlikely(vring_translate(dev, vq) < 0))
1197                         goto out;
1198
1199         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
1200         if (count == 0)
1201                 goto out;
1202
1203         if (vq_is_packed(dev))
1204                 nb_tx = virtio_dev_rx_packed(dev, vq, pkts, count);
1205         else
1206                 nb_tx = virtio_dev_rx_split(dev, vq, pkts, count);
1207
1208 out:
1209         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1210                 vhost_user_iotlb_rd_unlock(vq);
1211
1212 out_access_unlock:
1213         rte_spinlock_unlock(&vq->access_lock);
1214
1215         return nb_tx;
1216 }
1217
1218 uint16_t
1219 rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
1220         struct rte_mbuf **pkts, uint16_t count)
1221 {
1222         struct virtio_net *dev = get_device(vid);
1223
1224         if (!dev)
1225                 return 0;
1226
1227         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
1228                 VHOST_LOG_DATA(ERR,
1229                         "(%d) %s: built-in vhost net backend is disabled.\n",
1230                         dev->vid, __func__);
1231                 return 0;
1232         }
1233
1234         return virtio_dev_rx(dev, queue_id, pkts, count);
1235 }
1236
1237 static inline bool
1238 virtio_net_with_host_offload(struct virtio_net *dev)
1239 {
1240         if (dev->features &
1241                         ((1ULL << VIRTIO_NET_F_CSUM) |
1242                          (1ULL << VIRTIO_NET_F_HOST_ECN) |
1243                          (1ULL << VIRTIO_NET_F_HOST_TSO4) |
1244                          (1ULL << VIRTIO_NET_F_HOST_TSO6) |
1245                          (1ULL << VIRTIO_NET_F_HOST_UFO)))
1246                 return true;
1247
1248         return false;
1249 }
1250
1251 static void
1252 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
1253 {
1254         struct rte_ipv4_hdr *ipv4_hdr;
1255         struct rte_ipv6_hdr *ipv6_hdr;
1256         void *l3_hdr = NULL;
1257         struct rte_ether_hdr *eth_hdr;
1258         uint16_t ethertype;
1259
1260         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
1261
1262         m->l2_len = sizeof(struct rte_ether_hdr);
1263         ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
1264
1265         if (ethertype == RTE_ETHER_TYPE_VLAN) {
1266                 struct rte_vlan_hdr *vlan_hdr =
1267                         (struct rte_vlan_hdr *)(eth_hdr + 1);
1268
1269                 m->l2_len += sizeof(struct rte_vlan_hdr);
1270                 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
1271         }
1272
1273         l3_hdr = (char *)eth_hdr + m->l2_len;
1274
1275         switch (ethertype) {
1276         case RTE_ETHER_TYPE_IPV4:
1277                 ipv4_hdr = l3_hdr;
1278                 *l4_proto = ipv4_hdr->next_proto_id;
1279                 m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
1280                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1281                 m->ol_flags |= PKT_TX_IPV4;
1282                 break;
1283         case RTE_ETHER_TYPE_IPV6:
1284                 ipv6_hdr = l3_hdr;
1285                 *l4_proto = ipv6_hdr->proto;
1286                 m->l3_len = sizeof(struct rte_ipv6_hdr);
1287                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1288                 m->ol_flags |= PKT_TX_IPV6;
1289                 break;
1290         default:
1291                 m->l3_len = 0;
1292                 *l4_proto = 0;
1293                 *l4_hdr = NULL;
1294                 break;
1295         }
1296 }
1297
1298 static __rte_always_inline void
1299 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
1300 {
1301         uint16_t l4_proto = 0;
1302         void *l4_hdr = NULL;
1303         struct rte_tcp_hdr *tcp_hdr = NULL;
1304
1305         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
1306                 return;
1307
1308         parse_ethernet(m, &l4_proto, &l4_hdr);
1309         if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1310                 if (hdr->csum_start == (m->l2_len + m->l3_len)) {
1311                         switch (hdr->csum_offset) {
1312                         case (offsetof(struct rte_tcp_hdr, cksum)):
1313                                 if (l4_proto == IPPROTO_TCP)
1314                                         m->ol_flags |= PKT_TX_TCP_CKSUM;
1315                                 break;
1316                         case (offsetof(struct rte_udp_hdr, dgram_cksum)):
1317                                 if (l4_proto == IPPROTO_UDP)
1318                                         m->ol_flags |= PKT_TX_UDP_CKSUM;
1319                                 break;
1320                         case (offsetof(struct rte_sctp_hdr, cksum)):
1321                                 if (l4_proto == IPPROTO_SCTP)
1322                                         m->ol_flags |= PKT_TX_SCTP_CKSUM;
1323                                 break;
1324                         default:
1325                                 break;
1326                         }
1327                 }
1328         }
1329
1330         if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1331                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1332                 case VIRTIO_NET_HDR_GSO_TCPV4:
1333                 case VIRTIO_NET_HDR_GSO_TCPV6:
1334                         tcp_hdr = l4_hdr;
1335                         m->ol_flags |= PKT_TX_TCP_SEG;
1336                         m->tso_segsz = hdr->gso_size;
1337                         m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
1338                         break;
1339                 case VIRTIO_NET_HDR_GSO_UDP:
1340                         m->ol_flags |= PKT_TX_UDP_SEG;
1341                         m->tso_segsz = hdr->gso_size;
1342                         m->l4_len = sizeof(struct rte_udp_hdr);
1343                         break;
1344                 default:
1345                         VHOST_LOG_DATA(WARNING,
1346                                 "unsupported gso type %u.\n", hdr->gso_type);
1347                         break;
1348                 }
1349         }
1350 }
1351
1352 static __rte_noinline void
1353 copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr,
1354                 struct buf_vector *buf_vec)
1355 {
1356         uint64_t len;
1357         uint64_t remain = sizeof(struct virtio_net_hdr);
1358         uint64_t src;
1359         uint64_t dst = (uint64_t)(uintptr_t)hdr;
1360
1361         while (remain) {
1362                 len = RTE_MIN(remain, buf_vec->buf_len);
1363                 src = buf_vec->buf_addr;
1364                 rte_memcpy((void *)(uintptr_t)dst,
1365                                 (void *)(uintptr_t)src, len);
1366
1367                 remain -= len;
1368                 dst += len;
1369                 buf_vec++;
1370         }
1371 }
1372
1373 static __rte_always_inline int
1374 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
1375                   struct buf_vector *buf_vec, uint16_t nr_vec,
1376                   struct rte_mbuf *m, struct rte_mempool *mbuf_pool)
1377 {
1378         uint32_t buf_avail, buf_offset;
1379         uint64_t buf_addr, buf_iova, buf_len;
1380         uint32_t mbuf_avail, mbuf_offset;
1381         uint32_t cpy_len;
1382         struct rte_mbuf *cur = m, *prev = m;
1383         struct virtio_net_hdr tmp_hdr;
1384         struct virtio_net_hdr *hdr = NULL;
1385         /* A counter to avoid desc dead loop chain */
1386         uint16_t vec_idx = 0;
1387         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
1388         int error = 0;
1389
1390         buf_addr = buf_vec[vec_idx].buf_addr;
1391         buf_iova = buf_vec[vec_idx].buf_iova;
1392         buf_len = buf_vec[vec_idx].buf_len;
1393
1394         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
1395                 error = -1;
1396                 goto out;
1397         }
1398
1399         if (virtio_net_with_host_offload(dev)) {
1400                 if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
1401                         /*
1402                          * No luck, the virtio-net header doesn't fit
1403                          * in a contiguous virtual area.
1404                          */
1405                         copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec);
1406                         hdr = &tmp_hdr;
1407                 } else {
1408                         hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);
1409                 }
1410         }
1411
1412         /*
1413          * A virtio driver normally uses at least 2 desc buffers
1414          * for Tx: the first for storing the header, and others
1415          * for storing the data.
1416          */
1417         if (unlikely(buf_len < dev->vhost_hlen)) {
1418                 buf_offset = dev->vhost_hlen - buf_len;
1419                 vec_idx++;
1420                 buf_addr = buf_vec[vec_idx].buf_addr;
1421                 buf_iova = buf_vec[vec_idx].buf_iova;
1422                 buf_len = buf_vec[vec_idx].buf_len;
1423                 buf_avail  = buf_len - buf_offset;
1424         } else if (buf_len == dev->vhost_hlen) {
1425                 if (unlikely(++vec_idx >= nr_vec))
1426                         goto out;
1427                 buf_addr = buf_vec[vec_idx].buf_addr;
1428                 buf_iova = buf_vec[vec_idx].buf_iova;
1429                 buf_len = buf_vec[vec_idx].buf_len;
1430
1431                 buf_offset = 0;
1432                 buf_avail = buf_len;
1433         } else {
1434                 buf_offset = dev->vhost_hlen;
1435                 buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
1436         }
1437
1438         PRINT_PACKET(dev,
1439                         (uintptr_t)(buf_addr + buf_offset),
1440                         (uint32_t)buf_avail, 0);
1441
1442         mbuf_offset = 0;
1443         mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
1444         while (1) {
1445                 uint64_t hpa;
1446
1447                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
1448
1449                 /*
1450                  * A desc buf might across two host physical pages that are
1451                  * not continuous. In such case (gpa_to_hpa returns 0), data
1452                  * will be copied even though zero copy is enabled.
1453                  */
1454                 if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,
1455                                         buf_iova + buf_offset, cpy_len)))) {
1456                         cur->data_len = cpy_len;
1457                         cur->data_off = 0;
1458                         cur->buf_addr =
1459                                 (void *)(uintptr_t)(buf_addr + buf_offset);
1460                         cur->buf_iova = hpa;
1461
1462                         /*
1463                          * In zero copy mode, one mbuf can only reference data
1464                          * for one or partial of one desc buff.
1465                          */
1466                         mbuf_avail = cpy_len;
1467                 } else {
1468                         if (likely(cpy_len > MAX_BATCH_LEN ||
1469                                    vq->batch_copy_nb_elems >= vq->size ||
1470                                    (hdr && cur == m))) {
1471                                 rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
1472                                                                    mbuf_offset),
1473                                            (void *)((uintptr_t)(buf_addr +
1474                                                            buf_offset)),
1475                                            cpy_len);
1476                         } else {
1477                                 batch_copy[vq->batch_copy_nb_elems].dst =
1478                                         rte_pktmbuf_mtod_offset(cur, void *,
1479                                                                 mbuf_offset);
1480                                 batch_copy[vq->batch_copy_nb_elems].src =
1481                                         (void *)((uintptr_t)(buf_addr +
1482                                                                 buf_offset));
1483                                 batch_copy[vq->batch_copy_nb_elems].len =
1484                                         cpy_len;
1485                                 vq->batch_copy_nb_elems++;
1486                         }
1487                 }
1488
1489                 mbuf_avail  -= cpy_len;
1490                 mbuf_offset += cpy_len;
1491                 buf_avail -= cpy_len;
1492                 buf_offset += cpy_len;
1493
1494                 /* This buf reaches to its end, get the next one */
1495                 if (buf_avail == 0) {
1496                         if (++vec_idx >= nr_vec)
1497                                 break;
1498
1499                         buf_addr = buf_vec[vec_idx].buf_addr;
1500                         buf_iova = buf_vec[vec_idx].buf_iova;
1501                         buf_len = buf_vec[vec_idx].buf_len;
1502
1503                         buf_offset = 0;
1504                         buf_avail  = buf_len;
1505
1506                         PRINT_PACKET(dev, (uintptr_t)buf_addr,
1507                                         (uint32_t)buf_avail, 0);
1508                 }
1509
1510                 /*
1511                  * This mbuf reaches to its end, get a new one
1512                  * to hold more data.
1513                  */
1514                 if (mbuf_avail == 0) {
1515                         cur = rte_pktmbuf_alloc(mbuf_pool);
1516                         if (unlikely(cur == NULL)) {
1517                                 VHOST_LOG_DATA(ERR, "Failed to "
1518                                         "allocate memory for mbuf.\n");
1519                                 error = -1;
1520                                 goto out;
1521                         }
1522                         if (unlikely(dev->dequeue_zero_copy))
1523                                 rte_mbuf_refcnt_update(cur, 1);
1524
1525                         prev->next = cur;
1526                         prev->data_len = mbuf_offset;
1527                         m->nb_segs += 1;
1528                         m->pkt_len += mbuf_offset;
1529                         prev = cur;
1530
1531                         mbuf_offset = 0;
1532                         mbuf_avail  = cur->buf_len - RTE_PKTMBUF_HEADROOM;
1533                 }
1534         }
1535
1536         prev->data_len = mbuf_offset;
1537         m->pkt_len    += mbuf_offset;
1538
1539         if (hdr)
1540                 vhost_dequeue_offload(hdr, m);
1541
1542 out:
1543
1544         return error;
1545 }
1546
1547 static __rte_always_inline struct zcopy_mbuf *
1548 get_zmbuf(struct vhost_virtqueue *vq)
1549 {
1550         uint16_t i;
1551         uint16_t last;
1552         int tries = 0;
1553
1554         /* search [last_zmbuf_idx, zmbuf_size) */
1555         i = vq->last_zmbuf_idx;
1556         last = vq->zmbuf_size;
1557
1558 again:
1559         for (; i < last; i++) {
1560                 if (vq->zmbufs[i].in_use == 0) {
1561                         vq->last_zmbuf_idx = i + 1;
1562                         vq->zmbufs[i].in_use = 1;
1563                         return &vq->zmbufs[i];
1564                 }
1565         }
1566
1567         tries++;
1568         if (tries == 1) {
1569                 /* search [0, last_zmbuf_idx) */
1570                 i = 0;
1571                 last = vq->last_zmbuf_idx;
1572                 goto again;
1573         }
1574
1575         return NULL;
1576 }
1577
1578 static void
1579 virtio_dev_extbuf_free(void *addr __rte_unused, void *opaque)
1580 {
1581         rte_free(opaque);
1582 }
1583
1584 static int
1585 virtio_dev_extbuf_alloc(struct rte_mbuf *pkt, uint32_t size)
1586 {
1587         struct rte_mbuf_ext_shared_info *shinfo = NULL;
1588         uint32_t total_len = RTE_PKTMBUF_HEADROOM + size;
1589         uint16_t buf_len;
1590         rte_iova_t iova;
1591         void *buf;
1592
1593         /* Try to use pkt buffer to store shinfo to reduce the amount of memory
1594          * required, otherwise store shinfo in the new buffer.
1595          */
1596         if (rte_pktmbuf_tailroom(pkt) >= sizeof(*shinfo))
1597                 shinfo = rte_pktmbuf_mtod(pkt,
1598                                           struct rte_mbuf_ext_shared_info *);
1599         else {
1600                 total_len += sizeof(*shinfo) + sizeof(uintptr_t);
1601                 total_len = RTE_ALIGN_CEIL(total_len, sizeof(uintptr_t));
1602         }
1603
1604         if (unlikely(total_len > UINT16_MAX))
1605                 return -ENOSPC;
1606
1607         buf_len = total_len;
1608         buf = rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE);
1609         if (unlikely(buf == NULL))
1610                 return -ENOMEM;
1611
1612         /* Initialize shinfo */
1613         if (shinfo) {
1614                 shinfo->free_cb = virtio_dev_extbuf_free;
1615                 shinfo->fcb_opaque = buf;
1616                 rte_mbuf_ext_refcnt_set(shinfo, 1);
1617         } else {
1618                 shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len,
1619                                               virtio_dev_extbuf_free, buf);
1620                 if (unlikely(shinfo == NULL)) {
1621                         rte_free(buf);
1622                         VHOST_LOG_DATA(ERR, "Failed to init shinfo\n");
1623                         return -1;
1624                 }
1625         }
1626
1627         iova = rte_malloc_virt2iova(buf);
1628         rte_pktmbuf_attach_extbuf(pkt, buf, iova, buf_len, shinfo);
1629         rte_pktmbuf_reset_headroom(pkt);
1630
1631         return 0;
1632 }
1633
1634 /*
1635  * Allocate a host supported pktmbuf.
1636  */
1637 static __rte_always_inline struct rte_mbuf *
1638 virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct rte_mempool *mp,
1639                          uint32_t data_len)
1640 {
1641         struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp);
1642
1643         if (unlikely(pkt == NULL)) {
1644                 VHOST_LOG_DATA(ERR,
1645                         "Failed to allocate memory for mbuf.\n");
1646                 return NULL;
1647         }
1648
1649         if (rte_pktmbuf_tailroom(pkt) >= data_len)
1650                 return pkt;
1651
1652         /* attach an external buffer if supported */
1653         if (dev->extbuf && !virtio_dev_extbuf_alloc(pkt, data_len))
1654                 return pkt;
1655
1656         /* check if chained buffers are allowed */
1657         if (!dev->linearbuf)
1658                 return pkt;
1659
1660         /* Data doesn't fit into the buffer and the host supports
1661          * only linear buffers
1662          */
1663         rte_pktmbuf_free(pkt);
1664
1665         return NULL;
1666 }
1667
1668 static __rte_noinline uint16_t
1669 virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
1670         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1671 {
1672         uint16_t i;
1673         uint16_t free_entries;
1674
1675         if (unlikely(dev->dequeue_zero_copy)) {
1676                 struct zcopy_mbuf *zmbuf, *next;
1677
1678                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1679                      zmbuf != NULL; zmbuf = next) {
1680                         next = TAILQ_NEXT(zmbuf, next);
1681
1682                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1683                                 update_shadow_used_ring_split(vq,
1684                                                 zmbuf->desc_idx, 0);
1685                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1686                                 restore_mbuf(zmbuf->mbuf);
1687                                 rte_pktmbuf_free(zmbuf->mbuf);
1688                                 put_zmbuf(zmbuf);
1689                                 vq->nr_zmbuf -= 1;
1690                         }
1691                 }
1692
1693                 if (likely(vq->shadow_used_idx)) {
1694                         flush_shadow_used_ring_split(dev, vq);
1695                         vhost_vring_call_split(dev, vq);
1696                 }
1697         }
1698
1699         /*
1700          * The ordering between avail index and
1701          * desc reads needs to be enforced.
1702          */
1703         free_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) -
1704                         vq->last_avail_idx;
1705         if (free_entries == 0)
1706                 return 0;
1707
1708         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
1709
1710         VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
1711
1712         count = RTE_MIN(count, MAX_PKT_BURST);
1713         count = RTE_MIN(count, free_entries);
1714         VHOST_LOG_DATA(DEBUG, "(%d) about to dequeue %u buffers\n",
1715                         dev->vid, count);
1716
1717         for (i = 0; i < count; i++) {
1718                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
1719                 uint16_t head_idx;
1720                 uint32_t buf_len;
1721                 uint16_t nr_vec = 0;
1722                 int err;
1723
1724                 if (unlikely(fill_vec_buf_split(dev, vq,
1725                                                 vq->last_avail_idx + i,
1726                                                 &nr_vec, buf_vec,
1727                                                 &head_idx, &buf_len,
1728                                                 VHOST_ACCESS_RO) < 0))
1729                         break;
1730
1731                 if (likely(dev->dequeue_zero_copy == 0))
1732                         update_shadow_used_ring_split(vq, head_idx, 0);
1733
1734                 pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
1735                 if (unlikely(pkts[i] == NULL))
1736                         break;
1737
1738                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
1739                                 mbuf_pool);
1740                 if (unlikely(err)) {
1741                         rte_pktmbuf_free(pkts[i]);
1742                         break;
1743                 }
1744
1745                 if (unlikely(dev->dequeue_zero_copy)) {
1746                         struct zcopy_mbuf *zmbuf;
1747
1748                         zmbuf = get_zmbuf(vq);
1749                         if (!zmbuf) {
1750                                 rte_pktmbuf_free(pkts[i]);
1751                                 break;
1752                         }
1753                         zmbuf->mbuf = pkts[i];
1754                         zmbuf->desc_idx = head_idx;
1755
1756                         /*
1757                          * Pin lock the mbuf; we will check later to see
1758                          * whether the mbuf is freed (when we are the last
1759                          * user) or not. If that's the case, we then could
1760                          * update the used ring safely.
1761                          */
1762                         rte_mbuf_refcnt_update(pkts[i], 1);
1763
1764                         vq->nr_zmbuf += 1;
1765                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1766                 }
1767         }
1768         vq->last_avail_idx += i;
1769
1770         if (likely(dev->dequeue_zero_copy == 0)) {
1771                 do_data_copy_dequeue(vq);
1772                 if (unlikely(i < count))
1773                         vq->shadow_used_idx = i;
1774                 if (likely(vq->shadow_used_idx)) {
1775                         flush_shadow_used_ring_split(dev, vq);
1776                         vhost_vring_call_split(dev, vq);
1777                 }
1778         }
1779
1780         return i;
1781 }
1782
1783 static __rte_always_inline int
1784 vhost_reserve_avail_batch_packed(struct virtio_net *dev,
1785                                  struct vhost_virtqueue *vq,
1786                                  struct rte_mempool *mbuf_pool,
1787                                  struct rte_mbuf **pkts,
1788                                  uint16_t avail_idx,
1789                                  uintptr_t *desc_addrs,
1790                                  uint16_t *ids)
1791 {
1792         bool wrap = vq->avail_wrap_counter;
1793         struct vring_packed_desc *descs = vq->desc_packed;
1794         struct virtio_net_hdr *hdr;
1795         uint64_t lens[PACKED_BATCH_SIZE];
1796         uint64_t buf_lens[PACKED_BATCH_SIZE];
1797         uint32_t buf_offset = dev->vhost_hlen;
1798         uint16_t flags, i;
1799
1800         if (unlikely(avail_idx & PACKED_BATCH_MASK))
1801                 return -1;
1802         if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))
1803                 return -1;
1804
1805         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1806                 flags = descs[avail_idx + i].flags;
1807                 if (unlikely((wrap != !!(flags & VRING_DESC_F_AVAIL)) ||
1808                              (wrap == !!(flags & VRING_DESC_F_USED))  ||
1809                              (flags & PACKED_DESC_SINGLE_DEQUEUE_FLAG)))
1810                         return -1;
1811         }
1812
1813         rte_smp_rmb();
1814
1815         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1816                 lens[i] = descs[avail_idx + i].len;
1817
1818         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1819                 desc_addrs[i] = vhost_iova_to_vva(dev, vq,
1820                                                   descs[avail_idx + i].addr,
1821                                                   &lens[i], VHOST_ACCESS_RW);
1822         }
1823
1824         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1825                 if (unlikely((lens[i] != descs[avail_idx + i].len)))
1826                         return -1;
1827         }
1828
1829         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1830                 pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, lens[i]);
1831                 if (!pkts[i])
1832                         goto free_buf;
1833         }
1834
1835         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1836                 buf_lens[i] = pkts[i]->buf_len - pkts[i]->data_off;
1837
1838         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1839                 if (unlikely(buf_lens[i] < (lens[i] - buf_offset)))
1840                         goto free_buf;
1841         }
1842
1843         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1844                 pkts[i]->pkt_len = descs[avail_idx + i].len - buf_offset;
1845                 pkts[i]->data_len = pkts[i]->pkt_len;
1846                 ids[i] = descs[avail_idx + i].id;
1847         }
1848
1849         if (virtio_net_with_host_offload(dev)) {
1850                 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1851                         hdr = (struct virtio_net_hdr *)(desc_addrs[i]);
1852                         vhost_dequeue_offload(hdr, pkts[i]);
1853                 }
1854         }
1855
1856         return 0;
1857
1858 free_buf:
1859         for (i = 0; i < PACKED_BATCH_SIZE; i++)
1860                 rte_pktmbuf_free(pkts[i]);
1861
1862         return -1;
1863 }
1864
1865 static __rte_always_inline int
1866 virtio_dev_tx_batch_packed(struct virtio_net *dev,
1867                            struct vhost_virtqueue *vq,
1868                            struct rte_mempool *mbuf_pool,
1869                            struct rte_mbuf **pkts)
1870 {
1871         uint16_t avail_idx = vq->last_avail_idx;
1872         uint32_t buf_offset = dev->vhost_hlen;
1873         uintptr_t desc_addrs[PACKED_BATCH_SIZE];
1874         uint16_t ids[PACKED_BATCH_SIZE];
1875         uint16_t i;
1876
1877         if (vhost_reserve_avail_batch_packed(dev, vq, mbuf_pool, pkts,
1878                                              avail_idx, desc_addrs, ids))
1879                 return -1;
1880
1881         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1882                 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]);
1883
1884         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1885                 rte_memcpy(rte_pktmbuf_mtod_offset(pkts[i], void *, 0),
1886                            (void *)(uintptr_t)(desc_addrs[i] + buf_offset),
1887                            pkts[i]->pkt_len);
1888
1889         if (virtio_net_is_inorder(dev))
1890                 vhost_shadow_dequeue_batch_packed_inorder(vq,
1891                         ids[PACKED_BATCH_SIZE - 1]);
1892         else
1893                 vhost_shadow_dequeue_batch_packed(dev, vq, ids);
1894
1895         vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
1896
1897         return 0;
1898 }
1899
1900 static __rte_always_inline int
1901 vhost_dequeue_single_packed(struct virtio_net *dev,
1902                             struct vhost_virtqueue *vq,
1903                             struct rte_mempool *mbuf_pool,
1904                             struct rte_mbuf **pkts,
1905                             uint16_t *buf_id,
1906                             uint16_t *desc_count)
1907 {
1908         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1909         uint32_t buf_len;
1910         uint16_t nr_vec = 0;
1911         int err;
1912
1913         if (unlikely(fill_vec_buf_packed(dev, vq,
1914                                          vq->last_avail_idx, desc_count,
1915                                          buf_vec, &nr_vec,
1916                                          buf_id, &buf_len,
1917                                          VHOST_ACCESS_RO) < 0))
1918                 return -1;
1919
1920         *pkts = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
1921         if (unlikely(*pkts == NULL)) {
1922                 VHOST_LOG_DATA(ERR,
1923                         "Failed to allocate memory for mbuf.\n");
1924                 return -1;
1925         }
1926
1927         err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, *pkts,
1928                                 mbuf_pool);
1929         if (unlikely(err)) {
1930                 rte_pktmbuf_free(*pkts);
1931                 return -1;
1932         }
1933
1934         return 0;
1935 }
1936
1937 static __rte_always_inline int
1938 virtio_dev_tx_single_packed(struct virtio_net *dev,
1939                             struct vhost_virtqueue *vq,
1940                             struct rte_mempool *mbuf_pool,
1941                             struct rte_mbuf **pkts)
1942 {
1943
1944         uint16_t buf_id, desc_count;
1945
1946         if (vhost_dequeue_single_packed(dev, vq, mbuf_pool, pkts, &buf_id,
1947                                         &desc_count))
1948                 return -1;
1949
1950         if (virtio_net_is_inorder(dev))
1951                 vhost_shadow_dequeue_single_packed_inorder(vq, buf_id,
1952                                                            desc_count);
1953         else
1954                 vhost_shadow_dequeue_single_packed(vq, buf_id, desc_count);
1955
1956         vq_inc_last_avail_packed(vq, desc_count);
1957
1958         return 0;
1959 }
1960
1961 static __rte_always_inline int
1962 virtio_dev_tx_batch_packed_zmbuf(struct virtio_net *dev,
1963                                  struct vhost_virtqueue *vq,
1964                                  struct rte_mempool *mbuf_pool,
1965                                  struct rte_mbuf **pkts)
1966 {
1967         struct zcopy_mbuf *zmbufs[PACKED_BATCH_SIZE];
1968         uintptr_t desc_addrs[PACKED_BATCH_SIZE];
1969         uint16_t ids[PACKED_BATCH_SIZE];
1970         uint16_t i;
1971
1972         uint16_t avail_idx = vq->last_avail_idx;
1973
1974         if (vhost_reserve_avail_batch_packed(dev, vq, mbuf_pool, pkts,
1975                                              avail_idx, desc_addrs, ids))
1976                 return -1;
1977
1978         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1979                 zmbufs[i] = get_zmbuf(vq);
1980
1981         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1982                 if (!zmbufs[i])
1983                         goto free_pkt;
1984         }
1985
1986         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1987                 zmbufs[i]->mbuf = pkts[i];
1988                 zmbufs[i]->desc_idx = ids[i];
1989                 zmbufs[i]->desc_count = 1;
1990         }
1991
1992         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1993                 rte_mbuf_refcnt_update(pkts[i], 1);
1994
1995         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1996                 TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbufs[i], next);
1997
1998         vq->nr_zmbuf += PACKED_BATCH_SIZE;
1999         vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
2000
2001         return 0;
2002
2003 free_pkt:
2004         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
2005                 rte_pktmbuf_free(pkts[i]);
2006
2007         return -1;
2008 }
2009
2010 static __rte_always_inline int
2011 virtio_dev_tx_single_packed_zmbuf(struct virtio_net *dev,
2012                                   struct vhost_virtqueue *vq,
2013                                   struct rte_mempool *mbuf_pool,
2014                                   struct rte_mbuf **pkts)
2015 {
2016         uint16_t buf_id, desc_count;
2017         struct zcopy_mbuf *zmbuf;
2018
2019         if (vhost_dequeue_single_packed(dev, vq, mbuf_pool, pkts, &buf_id,
2020                                         &desc_count))
2021                 return -1;
2022
2023         zmbuf = get_zmbuf(vq);
2024         if (!zmbuf) {
2025                 rte_pktmbuf_free(*pkts);
2026                 return -1;
2027         }
2028         zmbuf->mbuf = *pkts;
2029         zmbuf->desc_idx = buf_id;
2030         zmbuf->desc_count = desc_count;
2031
2032         rte_mbuf_refcnt_update(*pkts, 1);
2033
2034         vq->nr_zmbuf += 1;
2035         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
2036
2037         vq_inc_last_avail_packed(vq, desc_count);
2038         return 0;
2039 }
2040
2041 static __rte_always_inline void
2042 free_zmbuf(struct vhost_virtqueue *vq)
2043 {
2044         struct zcopy_mbuf *next = NULL;
2045         struct zcopy_mbuf *zmbuf;
2046
2047         for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
2048              zmbuf != NULL; zmbuf = next) {
2049                 next = TAILQ_NEXT(zmbuf, next);
2050
2051                 uint16_t last_used_idx = vq->last_used_idx;
2052
2053                 if (mbuf_is_consumed(zmbuf->mbuf)) {
2054                         uint16_t flags;
2055                         flags = vq->desc_packed[last_used_idx].flags;
2056                         if (vq->used_wrap_counter) {
2057                                 flags |= VRING_DESC_F_USED;
2058                                 flags |= VRING_DESC_F_AVAIL;
2059                         } else {
2060                                 flags &= ~VRING_DESC_F_USED;
2061                                 flags &= ~VRING_DESC_F_AVAIL;
2062                         }
2063
2064                         vq->desc_packed[last_used_idx].id = zmbuf->desc_idx;
2065                         vq->desc_packed[last_used_idx].len = 0;
2066
2067                         rte_smp_wmb();
2068                         vq->desc_packed[last_used_idx].flags = flags;
2069
2070                         vq_inc_last_used_packed(vq, zmbuf->desc_count);
2071
2072                         TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
2073                         restore_mbuf(zmbuf->mbuf);
2074                         rte_pktmbuf_free(zmbuf->mbuf);
2075                         put_zmbuf(zmbuf);
2076                         vq->nr_zmbuf -= 1;
2077                 }
2078         }
2079 }
2080
2081 static __rte_noinline uint16_t
2082 virtio_dev_tx_packed_zmbuf(struct virtio_net *dev,
2083                            struct vhost_virtqueue *vq,
2084                            struct rte_mempool *mbuf_pool,
2085                            struct rte_mbuf **pkts,
2086                            uint32_t count)
2087 {
2088         uint32_t pkt_idx = 0;
2089         uint32_t remained = count;
2090
2091         free_zmbuf(vq);
2092
2093         do {
2094                 if (remained >= PACKED_BATCH_SIZE) {
2095                         if (!virtio_dev_tx_batch_packed_zmbuf(dev, vq,
2096                                 mbuf_pool, &pkts[pkt_idx])) {
2097                                 pkt_idx += PACKED_BATCH_SIZE;
2098                                 remained -= PACKED_BATCH_SIZE;
2099                                 continue;
2100                         }
2101                 }
2102
2103                 if (virtio_dev_tx_single_packed_zmbuf(dev, vq, mbuf_pool,
2104                                                       &pkts[pkt_idx]))
2105                         break;
2106                 pkt_idx++;
2107                 remained--;
2108
2109         } while (remained);
2110
2111         if (pkt_idx)
2112                 vhost_vring_call_packed(dev, vq);
2113
2114         return pkt_idx;
2115 }
2116
2117 static __rte_noinline uint16_t
2118 virtio_dev_tx_packed(struct virtio_net *dev,
2119                      struct vhost_virtqueue *vq,
2120                      struct rte_mempool *mbuf_pool,
2121                      struct rte_mbuf **pkts,
2122                      uint32_t count)
2123 {
2124         uint32_t pkt_idx = 0;
2125         uint32_t remained = count;
2126
2127         do {
2128                 rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
2129
2130                 if (remained >= PACKED_BATCH_SIZE) {
2131                         if (!virtio_dev_tx_batch_packed(dev, vq, mbuf_pool,
2132                                                         &pkts[pkt_idx])) {
2133                                 pkt_idx += PACKED_BATCH_SIZE;
2134                                 remained -= PACKED_BATCH_SIZE;
2135                                 continue;
2136                         }
2137                 }
2138
2139                 if (virtio_dev_tx_single_packed(dev, vq, mbuf_pool,
2140                                                 &pkts[pkt_idx]))
2141                         break;
2142                 pkt_idx++;
2143                 remained--;
2144
2145         } while (remained);
2146
2147         if (vq->shadow_used_idx) {
2148                 do_data_copy_dequeue(vq);
2149
2150                 vhost_flush_dequeue_shadow_packed(dev, vq);
2151                 vhost_vring_call_packed(dev, vq);
2152         }
2153
2154         return pkt_idx;
2155 }
2156
2157 uint16_t
2158 rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
2159         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
2160 {
2161         struct virtio_net *dev;
2162         struct rte_mbuf *rarp_mbuf = NULL;
2163         struct vhost_virtqueue *vq;
2164         int16_t success = 1;
2165
2166         dev = get_device(vid);
2167         if (!dev)
2168                 return 0;
2169
2170         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
2171                 VHOST_LOG_DATA(ERR,
2172                         "(%d) %s: built-in vhost net backend is disabled.\n",
2173                         dev->vid, __func__);
2174                 return 0;
2175         }
2176
2177         if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
2178                 VHOST_LOG_DATA(ERR,
2179                         "(%d) %s: invalid virtqueue idx %d.\n",
2180                         dev->vid, __func__, queue_id);
2181                 return 0;
2182         }
2183
2184         vq = dev->virtqueue[queue_id];
2185
2186         if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
2187                 return 0;
2188
2189         if (unlikely(vq->enabled == 0)) {
2190                 count = 0;
2191                 goto out_access_unlock;
2192         }
2193
2194         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
2195                 vhost_user_iotlb_rd_lock(vq);
2196
2197         if (unlikely(vq->access_ok == 0))
2198                 if (unlikely(vring_translate(dev, vq) < 0)) {
2199                         count = 0;
2200                         goto out;
2201                 }
2202
2203         /*
2204          * Construct a RARP broadcast packet, and inject it to the "pkts"
2205          * array, to looks like that guest actually send such packet.
2206          *
2207          * Check user_send_rarp() for more information.
2208          *
2209          * broadcast_rarp shares a cacheline in the virtio_net structure
2210          * with some fields that are accessed during enqueue and
2211          * __atomic_compare_exchange_n causes a write if performed compare
2212          * and exchange. This could result in false sharing between enqueue
2213          * and dequeue.
2214          *
2215          * Prevent unnecessary false sharing by reading broadcast_rarp first
2216          * and only performing compare and exchange if the read indicates it
2217          * is likely to be set.
2218          */
2219         if (unlikely(__atomic_load_n(&dev->broadcast_rarp, __ATOMIC_ACQUIRE) &&
2220                         __atomic_compare_exchange_n(&dev->broadcast_rarp,
2221                         &success, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED))) {
2222
2223                 rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
2224                 if (rarp_mbuf == NULL) {
2225                         VHOST_LOG_DATA(ERR, "Failed to make RARP packet.\n");
2226                         count = 0;
2227                         goto out;
2228                 }
2229                 count -= 1;
2230         }
2231
2232         if (vq_is_packed(dev)) {
2233                 if (unlikely(dev->dequeue_zero_copy))
2234                         count = virtio_dev_tx_packed_zmbuf(dev, vq, mbuf_pool,
2235                                                            pkts, count);
2236                 else
2237                         count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts,
2238                                                      count);
2239         } else
2240                 count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);
2241
2242 out:
2243         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
2244                 vhost_user_iotlb_rd_unlock(vq);
2245
2246 out_access_unlock:
2247         rte_spinlock_unlock(&vq->access_lock);
2248
2249         if (unlikely(rarp_mbuf != NULL)) {
2250                 /*
2251                  * Inject it to the head of "pkts" array, so that switch's mac
2252                  * learning table will get updated first.
2253                  */
2254                 memmove(&pkts[1], pkts, count * sizeof(struct rte_mbuf *));
2255                 pkts[0] = rarp_mbuf;
2256                 count += 1;
2257         }
2258
2259         return count;
2260 }