lib/librte_vhost/virtio_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdbool.h>
   7 #include <linux/virtio_net.h>
   8
   9 #include <rte_mbuf.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_ether.h>
  12 #include <rte_ip.h>
  13 #include <rte_vhost.h>
  14 #include <rte_tcp.h>
  15 #include <rte_udp.h>
  16 #include <rte_sctp.h>
  17 #include <rte_arp.h>
  18 #include <rte_spinlock.h>
  19 #include <rte_malloc.h>
  20
  21 #include "iotlb.h"
  22 #include "vhost.h"
  23
  24 #define MAX_PKT_BURST 32
  25
  26 #define MAX_BATCH_LEN 256
  27
  28 static  __rte_always_inline bool
  29 rxvq_is_mergeable(struct virtio_net *dev)
  30 {
  31         return dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF);
  32 }
  33
  34 static bool
  35 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
  36 {
  37         return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring;
  38 }
  39
  40 static __rte_always_inline void
  41 do_flush_shadow_used_ring_split(struct virtio_net *dev,
  42                         struct vhost_virtqueue *vq,
  43                         uint16_t to, uint16_t from, uint16_t size)
  44 {
  45         rte_memcpy(&vq->used->ring[to],
  46                         &vq->shadow_used_split[from],
  47                         size * sizeof(struct vring_used_elem));
  48         vhost_log_cache_used_vring(dev, vq,
  49                         offsetof(struct vring_used, ring[to]),
  50                         size * sizeof(struct vring_used_elem));
  51 }
  52
  53 static __rte_always_inline void
  54 flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
  55 {
  56         uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
  57
  58         if (used_idx + vq->shadow_used_idx <= vq->size) {
  59                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0,
  60                                           vq->shadow_used_idx);
  61         } else {
  62                 uint16_t size;
  63
  64                 /* update used ring interval [used_idx, vq->size] */
  65                 size = vq->size - used_idx;
  66                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size);
  67
  68                 /* update the left half used ring interval [0, left_size] */
  69                 do_flush_shadow_used_ring_split(dev, vq, 0, size,
  70                                           vq->shadow_used_idx - size);
  71         }
  72         vq->last_used_idx += vq->shadow_used_idx;
  73
  74         rte_smp_wmb();
  75
  76         vhost_log_cache_sync(dev, vq);
  77
  78         *(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx;
  79         vq->shadow_used_idx = 0;
  80         vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
  81                 sizeof(vq->used->idx));
  82 }
  83
  84 static __rte_always_inline void
  85 update_shadow_used_ring_split(struct vhost_virtqueue *vq,
  86                          uint16_t desc_idx, uint32_t len)
  87 {
  88         uint16_t i = vq->shadow_used_idx++;
  89
  90         vq->shadow_used_split[i].id  = desc_idx;
  91         vq->shadow_used_split[i].len = len;
  92 }
  93
  94 static __rte_always_inline void
  95 vhost_flush_enqueue_shadow_packed(struct virtio_net *dev,
  96                                   struct vhost_virtqueue *vq)
  97 {
  98         int i;
  99         uint16_t used_idx = vq->last_used_idx;
 100         uint16_t head_idx = vq->last_used_idx;
 101         uint16_t head_flags = 0;
 102
 103         /* Split loop in two to save memory barriers */
 104         for (i = 0; i < vq->shadow_used_idx; i++) {
 105                 vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id;
 106                 vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len;
 107
 108                 used_idx += vq->shadow_used_packed[i].count;
 109                 if (used_idx >= vq->size)
 110                         used_idx -= vq->size;
 111         }
 112
 113         rte_smp_wmb();
 114
 115         for (i = 0; i < vq->shadow_used_idx; i++) {
 116                 uint16_t flags;
 117
 118                 if (vq->shadow_used_packed[i].len)
 119                         flags = VRING_DESC_F_WRITE;
 120                 else
 121                         flags = 0;
 122
 123                 if (vq->used_wrap_counter) {
 124                         flags |= VRING_DESC_F_USED;
 125                         flags |= VRING_DESC_F_AVAIL;
 126                 } else {
 127                         flags &= ~VRING_DESC_F_USED;
 128                         flags &= ~VRING_DESC_F_AVAIL;
 129                 }
 130
 131                 if (i > 0) {
 132                         vq->desc_packed[vq->last_used_idx].flags = flags;
 133
 134                         vhost_log_cache_used_vring(dev, vq,
 135                                         vq->last_used_idx *
 136                                         sizeof(struct vring_packed_desc),
 137                                         sizeof(struct vring_packed_desc));
 138                 } else {
 139                         head_idx = vq->last_used_idx;
 140                         head_flags = flags;
 141                 }
 142
 143                 vq_inc_last_used_packed(vq, vq->shadow_used_packed[i].count);
 144         }
 145
 146         vq->desc_packed[head_idx].flags = head_flags;
 147
 148         vhost_log_cache_used_vring(dev, vq,
 149                                 head_idx *
 150                                 sizeof(struct vring_packed_desc),
 151                                 sizeof(struct vring_packed_desc));
 152
 153         vq->shadow_used_idx = 0;
 154         vhost_log_cache_sync(dev, vq);
 155 }
 156
 157 static __rte_always_inline void
 158 flush_shadow_used_ring_packed(struct virtio_net *dev,
 159                         struct vhost_virtqueue *vq)
 160 {
 161         int i;
 162         uint16_t used_idx = vq->last_used_idx;
 163         uint16_t head_idx = vq->last_used_idx;
 164         uint16_t head_flags = 0;
 165
 166         /* Split loop in two to save memory barriers */
 167         for (i = 0; i < vq->shadow_used_idx; i++) {
 168                 vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id;
 169                 vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len;
 170
 171                 used_idx += vq->shadow_used_packed[i].count;
 172                 if (used_idx >= vq->size)
 173                         used_idx -= vq->size;
 174         }
 175
 176         for (i = 0; i < vq->shadow_used_idx; i++) {
 177                 uint16_t flags;
 178
 179                 if (vq->shadow_used_packed[i].len)
 180                         flags = VRING_DESC_F_WRITE;
 181                 else
 182                         flags = 0;
 183
 184                 if (vq->used_wrap_counter) {
 185                         flags |= VRING_DESC_F_USED;
 186                         flags |= VRING_DESC_F_AVAIL;
 187                 } else {
 188                         flags &= ~VRING_DESC_F_USED;
 189                         flags &= ~VRING_DESC_F_AVAIL;
 190                 }
 191
 192                 if (i > 0) {
 193                         vq->desc_packed[vq->last_used_idx].flags = flags;
 194
 195                         vhost_log_cache_used_vring(dev, vq,
 196                                         vq->last_used_idx *
 197                                         sizeof(struct vring_packed_desc),
 198                                         sizeof(struct vring_packed_desc));
 199                 } else {
 200                         head_idx = vq->last_used_idx;
 201                         head_flags = flags;
 202                 }
 203
 204                 vq_inc_last_used_packed(vq, vq->shadow_used_packed[i].count);
 205         }
 206
 207         __atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
 208                          __ATOMIC_RELEASE);
 209
 210         vhost_log_cache_used_vring(dev, vq,
 211                                 head_idx *
 212                                 sizeof(struct vring_packed_desc),
 213                                 sizeof(struct vring_packed_desc));
 214
 215         vq->shadow_used_idx = 0;
 216         vhost_log_cache_sync(dev, vq);
 217 }
 218
 219 static __rte_always_inline void
 220 update_shadow_used_ring_packed(struct vhost_virtqueue *vq,
 221                          uint16_t desc_idx, uint32_t len, uint16_t count)
 222 {
 223         uint16_t i = vq->shadow_used_idx++;
 224
 225         vq->shadow_used_packed[i].id  = desc_idx;
 226         vq->shadow_used_packed[i].len = len;
 227         vq->shadow_used_packed[i].count = count;
 228 }
 229
 230 static inline void
 231 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
 232 {
 233         struct batch_copy_elem *elem = vq->batch_copy_elems;
 234         uint16_t count = vq->batch_copy_nb_elems;
 235         int i;
 236
 237         for (i = 0; i < count; i++) {
 238                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
 239                 vhost_log_cache_write_iova(dev, vq, elem[i].log_addr,
 240                                            elem[i].len);
 241                 PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
 242         }
 243
 244         vq->batch_copy_nb_elems = 0;
 245 }
 246
 247 static inline void
 248 do_data_copy_dequeue(struct vhost_virtqueue *vq)
 249 {
 250         struct batch_copy_elem *elem = vq->batch_copy_elems;
 251         uint16_t count = vq->batch_copy_nb_elems;
 252         int i;
 253
 254         for (i = 0; i < count; i++)
 255                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
 256
 257         vq->batch_copy_nb_elems = 0;
 258 }
 259
 260 static __rte_always_inline void
 261 vhost_shadow_enqueue_single_packed(struct virtio_net *dev,
 262                                    struct vhost_virtqueue *vq,
 263                                    uint32_t len[],
 264                                    uint16_t id[],
 265                                    uint16_t count[],
 266                                    uint16_t num_buffers)
 267 {
 268         uint16_t i;
 269         for (i = 0; i < num_buffers; i++) {
 270                 /* enqueue shadow flush action aligned with batch num */
 271                 if (!vq->shadow_used_idx)
 272                         vq->shadow_aligned_idx = vq->last_used_idx &
 273                                 PACKED_BATCH_MASK;
 274                 vq->shadow_used_packed[vq->shadow_used_idx].id  = id[i];
 275                 vq->shadow_used_packed[vq->shadow_used_idx].len = len[i];
 276                 vq->shadow_used_packed[vq->shadow_used_idx].count = count[i];
 277                 vq->shadow_aligned_idx += count[i];
 278                 vq->shadow_used_idx++;
 279         }
 280
 281         if (vq->shadow_aligned_idx >= PACKED_BATCH_SIZE) {
 282                 do_data_copy_enqueue(dev, vq);
 283                 vhost_flush_enqueue_shadow_packed(dev, vq);
 284         }
 285 }
 286
 287 /* avoid write operation when necessary, to lessen cache issues */
 288 #define ASSIGN_UNLESS_EQUAL(var, val) do {      \
 289         if ((var) != (val))                     \
 290                 (var) = (val);                  \
 291 } while (0)
 292
 293 static __rte_always_inline void
 294 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 295 {
 296         uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK;
 297
 298         if (m_buf->ol_flags & PKT_TX_TCP_SEG)
 299                 csum_l4 |= PKT_TX_TCP_CKSUM;
 300
 301         if (csum_l4) {
 302                 net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
 303                 net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len;
 304
 305                 switch (csum_l4) {
 306                 case PKT_TX_TCP_CKSUM:
 307                         net_hdr->csum_offset = (offsetof(struct rte_tcp_hdr,
 308                                                 cksum));
 309                         break;
 310                 case PKT_TX_UDP_CKSUM:
 311                         net_hdr->csum_offset = (offsetof(struct rte_udp_hdr,
 312                                                 dgram_cksum));
 313                         break;
 314                 case PKT_TX_SCTP_CKSUM:
 315                         net_hdr->csum_offset = (offsetof(struct rte_sctp_hdr,
 316                                                 cksum));
 317                         break;
 318                 }
 319         } else {
 320                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0);
 321                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0);
 322                 ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0);
 323         }
 324
 325         /* IP cksum verification cannot be bypassed, then calculate here */
 326         if (m_buf->ol_flags & PKT_TX_IP_CKSUM) {
 327                 struct rte_ipv4_hdr *ipv4_hdr;
 328
 329                 ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct rte_ipv4_hdr *,
 330                                                    m_buf->l2_len);
 331                 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
 332         }
 333
 334         if (m_buf->ol_flags & PKT_TX_TCP_SEG) {
 335                 if (m_buf->ol_flags & PKT_TX_IPV4)
 336                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
 337                 else
 338                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
 339                 net_hdr->gso_size = m_buf->tso_segsz;
 340                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len
 341                                         + m_buf->l4_len;
 342         } else if (m_buf->ol_flags & PKT_TX_UDP_SEG) {
 343                 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
 344                 net_hdr->gso_size = m_buf->tso_segsz;
 345                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len +
 346                         m_buf->l4_len;
 347         } else {
 348                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0);
 349                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0);
 350                 ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0);
 351         }
 352 }
 353
 354 static __rte_always_inline int
 355 map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 356                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 357                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 358 {
 359         uint16_t vec_id = *vec_idx;
 360
 361         while (desc_len) {
 362                 uint64_t desc_addr;
 363                 uint64_t desc_chunck_len = desc_len;
 364
 365                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 366                         return -1;
 367
 368                 desc_addr = vhost_iova_to_vva(dev, vq,
 369                                 desc_iova,
 370                                 &desc_chunck_len,
 371                                 perm);
 372                 if (unlikely(!desc_addr))
 373                         return -1;
 374
 375                 rte_prefetch0((void *)(uintptr_t)desc_addr);
 376
 377                 buf_vec[vec_id].buf_iova = desc_iova;
 378                 buf_vec[vec_id].buf_addr = desc_addr;
 379                 buf_vec[vec_id].buf_len  = desc_chunck_len;
 380
 381                 desc_len -= desc_chunck_len;
 382                 desc_iova += desc_chunck_len;
 383                 vec_id++;
 384         }
 385         *vec_idx = vec_id;
 386
 387         return 0;
 388 }
 389
 390 static __rte_always_inline int
 391 fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 392                          uint32_t avail_idx, uint16_t *vec_idx,
 393                          struct buf_vector *buf_vec, uint16_t *desc_chain_head,
 394                          uint32_t *desc_chain_len, uint8_t perm)
 395 {
 396         uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
 397         uint16_t vec_id = *vec_idx;
 398         uint32_t len    = 0;
 399         uint64_t dlen;
 400         uint32_t nr_descs = vq->size;
 401         uint32_t cnt    = 0;
 402         struct vring_desc *descs = vq->desc;
 403         struct vring_desc *idesc = NULL;
 404
 405         if (unlikely(idx >= vq->size))
 406                 return -1;
 407
 408         *desc_chain_head = idx;
 409
 410         if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
 411                 dlen = vq->desc[idx].len;
 412                 nr_descs = dlen / sizeof(struct vring_desc);
 413                 if (unlikely(nr_descs > vq->size))
 414                         return -1;
 415
 416                 descs = (struct vring_desc *)(uintptr_t)
 417                         vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
 418                                                 &dlen,
 419                                                 VHOST_ACCESS_RO);
 420                 if (unlikely(!descs))
 421                         return -1;
 422
 423                 if (unlikely(dlen < vq->desc[idx].len)) {
 424                         /*
 425                          * The indirect desc table is not contiguous
 426                          * in process VA space, we have to copy it.
 427                          */
 428                         idesc = vhost_alloc_copy_ind_table(dev, vq,
 429                                         vq->desc[idx].addr, vq->desc[idx].len);
 430                         if (unlikely(!idesc))
 431                                 return -1;
 432
 433                         descs = idesc;
 434                 }
 435
 436                 idx = 0;
 437         }
 438
 439         while (1) {
 440                 if (unlikely(idx >= nr_descs || cnt++ >= nr_descs)) {
 441                         free_ind_table(idesc);
 442                         return -1;
 443                 }
 444
 445                 len += descs[idx].len;
 446
 447                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 448                                                 descs[idx].addr, descs[idx].len,
 449                                                 perm))) {
 450                         free_ind_table(idesc);
 451                         return -1;
 452                 }
 453
 454                 if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
 455                         break;
 456
 457                 idx = descs[idx].next;
 458         }
 459
 460         *desc_chain_len = len;
 461         *vec_idx = vec_id;
 462
 463         if (unlikely(!!idesc))
 464                 free_ind_table(idesc);
 465
 466         return 0;
 467 }
 468
 469 /*
 470  * Returns -1 on fail, 0 on success
 471  */
 472 static inline int
 473 reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 474                                 uint32_t size, struct buf_vector *buf_vec,
 475                                 uint16_t *num_buffers, uint16_t avail_head,
 476                                 uint16_t *nr_vec)
 477 {
 478         uint16_t cur_idx;
 479         uint16_t vec_idx = 0;
 480         uint16_t max_tries, tries = 0;
 481
 482         uint16_t head_idx = 0;
 483         uint32_t len = 0;
 484
 485         *num_buffers = 0;
 486         cur_idx  = vq->last_avail_idx;
 487
 488         if (rxvq_is_mergeable(dev))
 489                 max_tries = vq->size - 1;
 490         else
 491                 max_tries = 1;
 492
 493         while (size > 0) {
 494                 if (unlikely(cur_idx == avail_head))
 495                         return -1;
 496                 /*
 497                  * if we tried all available ring items, and still
 498                  * can't get enough buf, it means something abnormal
 499                  * happened.
 500                  */
 501                 if (unlikely(++tries > max_tries))
 502                         return -1;
 503
 504                 if (unlikely(fill_vec_buf_split(dev, vq, cur_idx,
 505                                                 &vec_idx, buf_vec,
 506                                                 &head_idx, &len,
 507                                                 VHOST_ACCESS_RW) < 0))
 508                         return -1;
 509                 len = RTE_MIN(len, size);
 510                 update_shadow_used_ring_split(vq, head_idx, len);
 511                 size -= len;
 512
 513                 cur_idx++;
 514                 *num_buffers += 1;
 515         }
 516
 517         *nr_vec = vec_idx;
 518
 519         return 0;
 520 }
 521
 522 static __rte_always_inline int
 523 fill_vec_buf_packed_indirect(struct virtio_net *dev,
 524                         struct vhost_virtqueue *vq,
 525                         struct vring_packed_desc *desc, uint16_t *vec_idx,
 526                         struct buf_vector *buf_vec, uint32_t *len, uint8_t perm)
 527 {
 528         uint16_t i;
 529         uint32_t nr_descs;
 530         uint16_t vec_id = *vec_idx;
 531         uint64_t dlen;
 532         struct vring_packed_desc *descs, *idescs = NULL;
 533
 534         dlen = desc->len;
 535         descs = (struct vring_packed_desc *)(uintptr_t)
 536                 vhost_iova_to_vva(dev, vq, desc->addr, &dlen, VHOST_ACCESS_RO);
 537         if (unlikely(!descs))
 538                 return -1;
 539
 540         if (unlikely(dlen < desc->len)) {
 541                 /*
 542                  * The indirect desc table is not contiguous
 543                  * in process VA space, we have to copy it.
 544                  */
 545                 idescs = vhost_alloc_copy_ind_table(dev,
 546                                 vq, desc->addr, desc->len);
 547                 if (unlikely(!idescs))
 548                         return -1;
 549
 550                 descs = idescs;
 551         }
 552
 553         nr_descs =  desc->len / sizeof(struct vring_packed_desc);
 554         if (unlikely(nr_descs >= vq->size)) {
 555                 free_ind_table(idescs);
 556                 return -1;
 557         }
 558
 559         for (i = 0; i < nr_descs; i++) {
 560                 if (unlikely(vec_id >= BUF_VECTOR_MAX)) {
 561                         free_ind_table(idescs);
 562                         return -1;
 563                 }
 564
 565                 *len += descs[i].len;
 566                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 567                                                 descs[i].addr, descs[i].len,
 568                                                 perm)))
 569                         return -1;
 570         }
 571         *vec_idx = vec_id;
 572
 573         if (unlikely(!!idescs))
 574                 free_ind_table(idescs);
 575
 576         return 0;
 577 }
 578
 579 static __rte_always_inline int
 580 fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 581                                 uint16_t avail_idx, uint16_t *desc_count,
 582                                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 583                                 uint16_t *buf_id, uint32_t *len, uint8_t perm)
 584 {
 585         bool wrap_counter = vq->avail_wrap_counter;
 586         struct vring_packed_desc *descs = vq->desc_packed;
 587         uint16_t vec_id = *vec_idx;
 588
 589         if (avail_idx < vq->last_avail_idx)
 590                 wrap_counter ^= 1;
 591
 592         /*
 593          * Perform a load-acquire barrier in desc_is_avail to
 594          * enforce the ordering between desc flags and desc
 595          * content.
 596          */
 597         if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
 598                 return -1;
 599
 600         *desc_count = 0;
 601         *len = 0;
 602
 603         while (1) {
 604                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 605                         return -1;
 606
 607                 if (unlikely(*desc_count >= vq->size))
 608                         return -1;
 609
 610                 *desc_count += 1;
 611                 *buf_id = descs[avail_idx].id;
 612
 613                 if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) {
 614                         if (unlikely(fill_vec_buf_packed_indirect(dev, vq,
 615                                                         &descs[avail_idx],
 616                                                         &vec_id, buf_vec,
 617                                                         len, perm) < 0))
 618                                 return -1;
 619                 } else {
 620                         *len += descs[avail_idx].len;
 621
 622                         if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 623                                                         descs[avail_idx].addr,
 624                                                         descs[avail_idx].len,
 625                                                         perm)))
 626                                 return -1;
 627                 }
 628
 629                 if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0)
 630                         break;
 631
 632                 if (++avail_idx >= vq->size) {
 633                         avail_idx -= vq->size;
 634                         wrap_counter ^= 1;
 635                 }
 636         }
 637
 638         *vec_idx = vec_id;
 639
 640         return 0;
 641 }
 642
 643 /*
 644  * Returns -1 on fail, 0 on success
 645  */
 646 static inline int
 647 reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 648                                 uint32_t size, struct buf_vector *buf_vec,
 649                                 uint16_t *nr_vec, uint16_t *num_buffers,
 650                                 uint16_t *nr_descs)
 651 {
 652         uint16_t avail_idx;
 653         uint16_t vec_idx = 0;
 654         uint16_t max_tries, tries = 0;
 655
 656         uint16_t buf_id = 0;
 657         uint32_t len = 0;
 658         uint16_t desc_count;
 659
 660         *num_buffers = 0;
 661         avail_idx = vq->last_avail_idx;
 662
 663         if (rxvq_is_mergeable(dev))
 664                 max_tries = vq->size - 1;
 665         else
 666                 max_tries = 1;
 667
 668         while (size > 0) {
 669                 /*
 670                  * if we tried all available ring items, and still
 671                  * can't get enough buf, it means something abnormal
 672                  * happened.
 673                  */
 674                 if (unlikely(++tries > max_tries))
 675                         return -1;
 676
 677                 if (unlikely(fill_vec_buf_packed(dev, vq,
 678                                                 avail_idx, &desc_count,
 679                                                 buf_vec, &vec_idx,
 680                                                 &buf_id, &len,
 681                                                 VHOST_ACCESS_RW) < 0))
 682                         return -1;
 683
 684                 len = RTE_MIN(len, size);
 685                 update_shadow_used_ring_packed(vq, buf_id, len, desc_count);
 686                 size -= len;
 687
 688                 avail_idx += desc_count;
 689                 if (avail_idx >= vq->size)
 690                         avail_idx -= vq->size;
 691
 692                 *nr_descs += desc_count;
 693                 *num_buffers += 1;
 694         }
 695
 696         *nr_vec = vec_idx;
 697
 698         return 0;
 699 }
 700
 701 static __rte_noinline void
 702 copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 703                 struct buf_vector *buf_vec,
 704                 struct virtio_net_hdr_mrg_rxbuf *hdr)
 705 {
 706         uint64_t len;
 707         uint64_t remain = dev->vhost_hlen;
 708         uint64_t src = (uint64_t)(uintptr_t)hdr, dst;
 709         uint64_t iova = buf_vec->buf_iova;
 710
 711         while (remain) {
 712                 len = RTE_MIN(remain,
 713                                 buf_vec->buf_len);
 714                 dst = buf_vec->buf_addr;
 715                 rte_memcpy((void *)(uintptr_t)dst,
 716                                 (void *)(uintptr_t)src,
 717                                 len);
 718
 719                 PRINT_PACKET(dev, (uintptr_t)dst,
 720                                 (uint32_t)len, 0);
 721                 vhost_log_cache_write_iova(dev, vq,
 722                                 iova, len);
 723
 724                 remain -= len;
 725                 iova += len;
 726                 src += len;
 727                 buf_vec++;
 728         }
 729 }
 730
 731 static __rte_always_inline int
 732 copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 733                             struct rte_mbuf *m, struct buf_vector *buf_vec,
 734                             uint16_t nr_vec, uint16_t num_buffers)
 735 {
 736         uint32_t vec_idx = 0;
 737         uint32_t mbuf_offset, mbuf_avail;
 738         uint32_t buf_offset, buf_avail;
 739         uint64_t buf_addr, buf_iova, buf_len;
 740         uint32_t cpy_len;
 741         uint64_t hdr_addr;
 742         struct rte_mbuf *hdr_mbuf;
 743         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 744         struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
 745         int error = 0;
 746
 747         if (unlikely(m == NULL)) {
 748                 error = -1;
 749                 goto out;
 750         }
 751
 752         buf_addr = buf_vec[vec_idx].buf_addr;
 753         buf_iova = buf_vec[vec_idx].buf_iova;
 754         buf_len = buf_vec[vec_idx].buf_len;
 755
 756         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 757                 error = -1;
 758                 goto out;
 759         }
 760
 761         hdr_mbuf = m;
 762         hdr_addr = buf_addr;
 763         if (unlikely(buf_len < dev->vhost_hlen))
 764                 hdr = &tmp_hdr;
 765         else
 766                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
 767
 768         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
 769                 dev->vid, num_buffers);
 770
 771         if (unlikely(buf_len < dev->vhost_hlen)) {
 772                 buf_offset = dev->vhost_hlen - buf_len;
 773                 vec_idx++;
 774                 buf_addr = buf_vec[vec_idx].buf_addr;
 775                 buf_iova = buf_vec[vec_idx].buf_iova;
 776                 buf_len = buf_vec[vec_idx].buf_len;
 777                 buf_avail = buf_len - buf_offset;
 778         } else {
 779                 buf_offset = dev->vhost_hlen;
 780                 buf_avail = buf_len - dev->vhost_hlen;
 781         }
 782
 783         mbuf_avail  = rte_pktmbuf_data_len(m);
 784         mbuf_offset = 0;
 785         while (mbuf_avail != 0 || m->next != NULL) {
 786                 /* done with current buf, get the next one */
 787                 if (buf_avail == 0) {
 788                         vec_idx++;
 789                         if (unlikely(vec_idx >= nr_vec)) {
 790                                 error = -1;
 791                                 goto out;
 792                         }
 793
 794                         buf_addr = buf_vec[vec_idx].buf_addr;
 795                         buf_iova = buf_vec[vec_idx].buf_iova;
 796                         buf_len = buf_vec[vec_idx].buf_len;
 797
 798                         buf_offset = 0;
 799                         buf_avail  = buf_len;
 800                 }
 801
 802                 /* done with current mbuf, get the next one */
 803                 if (mbuf_avail == 0) {
 804                         m = m->next;
 805
 806                         mbuf_offset = 0;
 807                         mbuf_avail  = rte_pktmbuf_data_len(m);
 808                 }
 809
 810                 if (hdr_addr) {
 811                         virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
 812                         if (rxvq_is_mergeable(dev))
 813                                 ASSIGN_UNLESS_EQUAL(hdr->num_buffers,
 814                                                 num_buffers);
 815
 816                         if (unlikely(hdr == &tmp_hdr)) {
 817                                 copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr);
 818                         } else {
 819                                 PRINT_PACKET(dev, (uintptr_t)hdr_addr,
 820                                                 dev->vhost_hlen, 0);
 821                                 vhost_log_cache_write_iova(dev, vq,
 822                                                 buf_vec[0].buf_iova,
 823                                                 dev->vhost_hlen);
 824                         }
 825
 826                         hdr_addr = 0;
 827                 }
 828
 829                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 830
 831                 if (likely(cpy_len > MAX_BATCH_LEN ||
 832                                         vq->batch_copy_nb_elems >= vq->size)) {
 833                         rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),
 834                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 835                                 cpy_len);
 836                         vhost_log_cache_write_iova(dev, vq,
 837                                                    buf_iova + buf_offset,
 838                                                    cpy_len);
 839                         PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),
 840                                 cpy_len, 0);
 841                 } else {
 842                         batch_copy[vq->batch_copy_nb_elems].dst =
 843                                 (void *)((uintptr_t)(buf_addr + buf_offset));
 844                         batch_copy[vq->batch_copy_nb_elems].src =
 845                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
 846                         batch_copy[vq->batch_copy_nb_elems].log_addr =
 847                                 buf_iova + buf_offset;
 848                         batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 849                         vq->batch_copy_nb_elems++;
 850                 }
 851
 852                 mbuf_avail  -= cpy_len;
 853                 mbuf_offset += cpy_len;
 854                 buf_avail  -= cpy_len;
 855                 buf_offset += cpy_len;
 856         }
 857
 858 out:
 859
 860         return error;
 861 }
 862
 863 static __rte_always_inline int
 864 vhost_enqueue_single_packed(struct virtio_net *dev,
 865                             struct vhost_virtqueue *vq,
 866                             struct rte_mbuf *pkt,
 867                             struct buf_vector *buf_vec,
 868                             uint16_t *nr_descs)
 869 {
 870         uint16_t nr_vec = 0;
 871         uint16_t avail_idx = vq->last_avail_idx;
 872         uint16_t max_tries, tries = 0;
 873         uint16_t buf_id = 0;
 874         uint32_t len = 0;
 875         uint16_t desc_count;
 876         uint32_t size = pkt->pkt_len + dev->vhost_hlen;
 877         uint16_t num_buffers = 0;
 878         uint32_t buffer_len[vq->size];
 879         uint16_t buffer_buf_id[vq->size];
 880         uint16_t buffer_desc_count[vq->size];
 881
 882         if (rxvq_is_mergeable(dev))
 883                 max_tries = vq->size - 1;
 884         else
 885                 max_tries = 1;
 886
 887         while (size > 0) {
 888                 /*
 889                  * if we tried all available ring items, and still
 890                  * can't get enough buf, it means something abnormal
 891                  * happened.
 892                  */
 893                 if (unlikely(++tries > max_tries))
 894                         return -1;
 895
 896                 if (unlikely(fill_vec_buf_packed(dev, vq,
 897                                                 avail_idx, &desc_count,
 898                                                 buf_vec, &nr_vec,
 899                                                 &buf_id, &len,
 900                                                 VHOST_ACCESS_RW) < 0))
 901                         return -1;
 902
 903                 len = RTE_MIN(len, size);
 904                 size -= len;
 905
 906                 buffer_len[num_buffers] = len;
 907                 buffer_buf_id[num_buffers] = buf_id;
 908                 buffer_desc_count[num_buffers] = desc_count;
 909                 num_buffers += 1;
 910
 911                 *nr_descs += desc_count;
 912                 avail_idx += desc_count;
 913                 if (avail_idx >= vq->size)
 914                         avail_idx -= vq->size;
 915         }
 916
 917         if (copy_mbuf_to_desc(dev, vq, pkt, buf_vec, nr_vec, num_buffers) < 0)
 918                 return -1;
 919
 920         vhost_shadow_enqueue_single_packed(dev, vq, buffer_len, buffer_buf_id,
 921                                            buffer_desc_count, num_buffers);
 922
 923         return 0;
 924 }
 925
 926 static __rte_noinline uint32_t
 927 virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 928         struct rte_mbuf **pkts, uint32_t count)
 929 {
 930         uint32_t pkt_idx = 0;
 931         uint16_t num_buffers;
 932         struct buf_vector buf_vec[BUF_VECTOR_MAX];
 933         uint16_t avail_head;
 934
 935         avail_head = *((volatile uint16_t *)&vq->avail->idx);
 936
 937         /*
 938          * The ordering between avail index and
 939          * desc reads needs to be enforced.
 940          */
 941         rte_smp_rmb();
 942
 943         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
 944
 945         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 946                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
 947                 uint16_t nr_vec = 0;
 948
 949                 if (unlikely(reserve_avail_buf_split(dev, vq,
 950                                                 pkt_len, buf_vec, &num_buffers,
 951                                                 avail_head, &nr_vec) < 0)) {
 952                         VHOST_LOG_DEBUG(VHOST_DATA,
 953                                 "(%d) failed to get enough desc from vring\n",
 954                                 dev->vid);
 955                         vq->shadow_used_idx -= num_buffers;
 956                         break;
 957                 }
 958
 959                 VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
 960                         dev->vid, vq->last_avail_idx,
 961                         vq->last_avail_idx + num_buffers);
 962
 963                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
 964                                                 buf_vec, nr_vec,
 965                                                 num_buffers) < 0) {
 966                         vq->shadow_used_idx -= num_buffers;
 967                         break;
 968                 }
 969
 970                 vq->last_avail_idx += num_buffers;
 971         }
 972
 973         do_data_copy_enqueue(dev, vq);
 974
 975         if (likely(vq->shadow_used_idx)) {
 976                 flush_shadow_used_ring_split(dev, vq);
 977                 vhost_vring_call_split(dev, vq);
 978         }
 979
 980         return pkt_idx;
 981 }
 982
 983 static __rte_unused int
 984 virtio_dev_rx_batch_packed(struct virtio_net *dev,
 985                            struct vhost_virtqueue *vq,
 986                            struct rte_mbuf **pkts)
 987 {
 988         bool wrap_counter = vq->avail_wrap_counter;
 989         struct vring_packed_desc *descs = vq->desc_packed;
 990         uint16_t avail_idx = vq->last_avail_idx;
 991         uint64_t desc_addrs[PACKED_BATCH_SIZE];
 992         struct virtio_net_hdr_mrg_rxbuf *hdrs[PACKED_BATCH_SIZE];
 993         uint32_t buf_offset = dev->vhost_hlen;
 994         uint64_t lens[PACKED_BATCH_SIZE];
 995         uint16_t i;
 996
 997         if (unlikely(avail_idx & PACKED_BATCH_MASK))
 998                 return -1;
 999
1000         if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))
1001                 return -1;
1002
1003         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1004                 if (unlikely(pkts[i]->next != NULL))
1005                         return -1;
1006                 if (unlikely(!desc_is_avail(&descs[avail_idx + i],
1007                                             wrap_counter)))
1008                         return -1;
1009         }
1010
1011         rte_smp_rmb();
1012
1013         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1014                 lens[i] = descs[avail_idx + i].len;
1015
1016         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1017                 if (unlikely(pkts[i]->pkt_len > (lens[i] - buf_offset)))
1018                         return -1;
1019         }
1020
1021         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1022                 desc_addrs[i] = vhost_iova_to_vva(dev, vq,
1023                                                   descs[avail_idx + i].addr,
1024                                                   &lens[i],
1025                                                   VHOST_ACCESS_RW);
1026
1027         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1028                 if (unlikely(lens[i] != descs[avail_idx + i].len))
1029                         return -1;
1030         }
1031
1032         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1033                 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]);
1034                 hdrs[i] = (struct virtio_net_hdr_mrg_rxbuf *)
1035                                         (uintptr_t)desc_addrs[i];
1036                 lens[i] = pkts[i]->pkt_len + dev->vhost_hlen;
1037         }
1038
1039         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1040                 virtio_enqueue_offload(pkts[i], &hdrs[i]->hdr);
1041
1042         vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
1043
1044         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1045                 rte_memcpy((void *)(uintptr_t)(desc_addrs[i] + buf_offset),
1046                            rte_pktmbuf_mtod_offset(pkts[i], void *, 0),
1047                            pkts[i]->pkt_len);
1048         }
1049
1050         return 0;
1051 }
1052
1053 static __rte_unused int16_t
1054 virtio_dev_rx_single_packed(struct virtio_net *dev,
1055                             struct vhost_virtqueue *vq,
1056                             struct rte_mbuf *pkt)
1057 {
1058         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1059         uint16_t nr_descs = 0;
1060
1061         rte_smp_rmb();
1062         if (unlikely(vhost_enqueue_single_packed(dev, vq, pkt, buf_vec,
1063                                                  &nr_descs) < 0)) {
1064                 VHOST_LOG_DEBUG(VHOST_DATA,
1065                                 "(%d) failed to get enough desc from vring\n",
1066                                 dev->vid);
1067                 return -1;
1068         }
1069
1070         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
1071                         dev->vid, vq->last_avail_idx,
1072                         vq->last_avail_idx + nr_descs);
1073
1074         vq_inc_last_avail_packed(vq, nr_descs);
1075
1076         return 0;
1077 }
1078
1079 static __rte_noinline uint32_t
1080 virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
1081         struct rte_mbuf **pkts, uint32_t count)
1082 {
1083         uint32_t pkt_idx = 0;
1084         uint16_t num_buffers;
1085         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1086
1087         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
1088                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
1089                 uint16_t nr_vec = 0;
1090                 uint16_t nr_descs = 0;
1091
1092                 if (unlikely(reserve_avail_buf_packed(dev, vq,
1093                                                 pkt_len, buf_vec, &nr_vec,
1094                                                 &num_buffers, &nr_descs) < 0)) {
1095                         VHOST_LOG_DEBUG(VHOST_DATA,
1096                                 "(%d) failed to get enough desc from vring\n",
1097                                 dev->vid);
1098                         vq->shadow_used_idx -= num_buffers;
1099                         break;
1100                 }
1101
1102                 VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
1103                         dev->vid, vq->last_avail_idx,
1104                         vq->last_avail_idx + num_buffers);
1105
1106                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
1107                                                 buf_vec, nr_vec,
1108                                                 num_buffers) < 0) {
1109                         vq->shadow_used_idx -= num_buffers;
1110                         break;
1111                 }
1112
1113                 vq_inc_last_avail_packed(vq, nr_descs);
1114         }
1115
1116         do_data_copy_enqueue(dev, vq);
1117
1118         if (likely(vq->shadow_used_idx)) {
1119                 vhost_flush_enqueue_shadow_packed(dev, vq);
1120                 vhost_vring_call_packed(dev, vq);
1121         }
1122
1123         return pkt_idx;
1124 }
1125
1126 static __rte_always_inline uint32_t
1127 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
1128         struct rte_mbuf **pkts, uint32_t count)
1129 {
1130         struct vhost_virtqueue *vq;
1131         uint32_t nb_tx = 0;
1132
1133         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1134         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
1135                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
1136                         dev->vid, __func__, queue_id);
1137                 return 0;
1138         }
1139
1140         vq = dev->virtqueue[queue_id];
1141
1142         rte_spinlock_lock(&vq->access_lock);
1143
1144         if (unlikely(vq->enabled == 0))
1145                 goto out_access_unlock;
1146
1147         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1148                 vhost_user_iotlb_rd_lock(vq);
1149
1150         if (unlikely(vq->access_ok == 0))
1151                 if (unlikely(vring_translate(dev, vq) < 0))
1152                         goto out;
1153
1154         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
1155         if (count == 0)
1156                 goto out;
1157
1158         if (vq_is_packed(dev))
1159                 nb_tx = virtio_dev_rx_packed(dev, vq, pkts, count);
1160         else
1161                 nb_tx = virtio_dev_rx_split(dev, vq, pkts, count);
1162
1163 out:
1164         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1165                 vhost_user_iotlb_rd_unlock(vq);
1166
1167 out_access_unlock:
1168         rte_spinlock_unlock(&vq->access_lock);
1169
1170         return nb_tx;
1171 }
1172
1173 uint16_t
1174 rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
1175         struct rte_mbuf **pkts, uint16_t count)
1176 {
1177         struct virtio_net *dev = get_device(vid);
1178
1179         if (!dev)
1180                 return 0;
1181
1182         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
1183                 RTE_LOG(ERR, VHOST_DATA,
1184                         "(%d) %s: built-in vhost net backend is disabled.\n",
1185                         dev->vid, __func__);
1186                 return 0;
1187         }
1188
1189         return virtio_dev_rx(dev, queue_id, pkts, count);
1190 }
1191
1192 static inline bool
1193 virtio_net_with_host_offload(struct virtio_net *dev)
1194 {
1195         if (dev->features &
1196                         ((1ULL << VIRTIO_NET_F_CSUM) |
1197                          (1ULL << VIRTIO_NET_F_HOST_ECN) |
1198                          (1ULL << VIRTIO_NET_F_HOST_TSO4) |
1199                          (1ULL << VIRTIO_NET_F_HOST_TSO6) |
1200                          (1ULL << VIRTIO_NET_F_HOST_UFO)))
1201                 return true;
1202
1203         return false;
1204 }
1205
1206 static void
1207 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
1208 {
1209         struct rte_ipv4_hdr *ipv4_hdr;
1210         struct rte_ipv6_hdr *ipv6_hdr;
1211         void *l3_hdr = NULL;
1212         struct rte_ether_hdr *eth_hdr;
1213         uint16_t ethertype;
1214
1215         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
1216
1217         m->l2_len = sizeof(struct rte_ether_hdr);
1218         ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
1219
1220         if (ethertype == RTE_ETHER_TYPE_VLAN) {
1221                 struct rte_vlan_hdr *vlan_hdr =
1222                         (struct rte_vlan_hdr *)(eth_hdr + 1);
1223
1224                 m->l2_len += sizeof(struct rte_vlan_hdr);
1225                 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
1226         }
1227
1228         l3_hdr = (char *)eth_hdr + m->l2_len;
1229
1230         switch (ethertype) {
1231         case RTE_ETHER_TYPE_IPV4:
1232                 ipv4_hdr = l3_hdr;
1233                 *l4_proto = ipv4_hdr->next_proto_id;
1234                 m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
1235                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1236                 m->ol_flags |= PKT_TX_IPV4;
1237                 break;
1238         case RTE_ETHER_TYPE_IPV6:
1239                 ipv6_hdr = l3_hdr;
1240                 *l4_proto = ipv6_hdr->proto;
1241                 m->l3_len = sizeof(struct rte_ipv6_hdr);
1242                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1243                 m->ol_flags |= PKT_TX_IPV6;
1244                 break;
1245         default:
1246                 m->l3_len = 0;
1247                 *l4_proto = 0;
1248                 *l4_hdr = NULL;
1249                 break;
1250         }
1251 }
1252
1253 static __rte_always_inline void
1254 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
1255 {
1256         uint16_t l4_proto = 0;
1257         void *l4_hdr = NULL;
1258         struct rte_tcp_hdr *tcp_hdr = NULL;
1259
1260         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
1261                 return;
1262
1263         parse_ethernet(m, &l4_proto, &l4_hdr);
1264         if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1265                 if (hdr->csum_start == (m->l2_len + m->l3_len)) {
1266                         switch (hdr->csum_offset) {
1267                         case (offsetof(struct rte_tcp_hdr, cksum)):
1268                                 if (l4_proto == IPPROTO_TCP)
1269                                         m->ol_flags |= PKT_TX_TCP_CKSUM;
1270                                 break;
1271                         case (offsetof(struct rte_udp_hdr, dgram_cksum)):
1272                                 if (l4_proto == IPPROTO_UDP)
1273                                         m->ol_flags |= PKT_TX_UDP_CKSUM;
1274                                 break;
1275                         case (offsetof(struct rte_sctp_hdr, cksum)):
1276                                 if (l4_proto == IPPROTO_SCTP)
1277                                         m->ol_flags |= PKT_TX_SCTP_CKSUM;
1278                                 break;
1279                         default:
1280                                 break;
1281                         }
1282                 }
1283         }
1284
1285         if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1286                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1287                 case VIRTIO_NET_HDR_GSO_TCPV4:
1288                 case VIRTIO_NET_HDR_GSO_TCPV6:
1289                         tcp_hdr = l4_hdr;
1290                         m->ol_flags |= PKT_TX_TCP_SEG;
1291                         m->tso_segsz = hdr->gso_size;
1292                         m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
1293                         break;
1294                 case VIRTIO_NET_HDR_GSO_UDP:
1295                         m->ol_flags |= PKT_TX_UDP_SEG;
1296                         m->tso_segsz = hdr->gso_size;
1297                         m->l4_len = sizeof(struct rte_udp_hdr);
1298                         break;
1299                 default:
1300                         RTE_LOG(WARNING, VHOST_DATA,
1301                                 "unsupported gso type %u.\n", hdr->gso_type);
1302                         break;
1303                 }
1304         }
1305 }
1306
1307 static __rte_noinline void
1308 copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr,
1309                 struct buf_vector *buf_vec)
1310 {
1311         uint64_t len;
1312         uint64_t remain = sizeof(struct virtio_net_hdr);
1313         uint64_t src;
1314         uint64_t dst = (uint64_t)(uintptr_t)hdr;
1315
1316         while (remain) {
1317                 len = RTE_MIN(remain, buf_vec->buf_len);
1318                 src = buf_vec->buf_addr;
1319                 rte_memcpy((void *)(uintptr_t)dst,
1320                                 (void *)(uintptr_t)src, len);
1321
1322                 remain -= len;
1323                 dst += len;
1324                 buf_vec++;
1325         }
1326 }
1327
1328 static __rte_always_inline int
1329 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
1330                   struct buf_vector *buf_vec, uint16_t nr_vec,
1331                   struct rte_mbuf *m, struct rte_mempool *mbuf_pool)
1332 {
1333         uint32_t buf_avail, buf_offset;
1334         uint64_t buf_addr, buf_iova, buf_len;
1335         uint32_t mbuf_avail, mbuf_offset;
1336         uint32_t cpy_len;
1337         struct rte_mbuf *cur = m, *prev = m;
1338         struct virtio_net_hdr tmp_hdr;
1339         struct virtio_net_hdr *hdr = NULL;
1340         /* A counter to avoid desc dead loop chain */
1341         uint16_t vec_idx = 0;
1342         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
1343         int error = 0;
1344
1345         buf_addr = buf_vec[vec_idx].buf_addr;
1346         buf_iova = buf_vec[vec_idx].buf_iova;
1347         buf_len = buf_vec[vec_idx].buf_len;
1348
1349         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
1350                 error = -1;
1351                 goto out;
1352         }
1353
1354         if (virtio_net_with_host_offload(dev)) {
1355                 if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
1356                         /*
1357                          * No luck, the virtio-net header doesn't fit
1358                          * in a contiguous virtual area.
1359                          */
1360                         copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec);
1361                         hdr = &tmp_hdr;
1362                 } else {
1363                         hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);
1364                 }
1365         }
1366
1367         /*
1368          * A virtio driver normally uses at least 2 desc buffers
1369          * for Tx: the first for storing the header, and others
1370          * for storing the data.
1371          */
1372         if (unlikely(buf_len < dev->vhost_hlen)) {
1373                 buf_offset = dev->vhost_hlen - buf_len;
1374                 vec_idx++;
1375                 buf_addr = buf_vec[vec_idx].buf_addr;
1376                 buf_iova = buf_vec[vec_idx].buf_iova;
1377                 buf_len = buf_vec[vec_idx].buf_len;
1378                 buf_avail  = buf_len - buf_offset;
1379         } else if (buf_len == dev->vhost_hlen) {
1380                 if (unlikely(++vec_idx >= nr_vec))
1381                         goto out;
1382                 buf_addr = buf_vec[vec_idx].buf_addr;
1383                 buf_iova = buf_vec[vec_idx].buf_iova;
1384                 buf_len = buf_vec[vec_idx].buf_len;
1385
1386                 buf_offset = 0;
1387                 buf_avail = buf_len;
1388         } else {
1389                 buf_offset = dev->vhost_hlen;
1390                 buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
1391         }
1392
1393         PRINT_PACKET(dev,
1394                         (uintptr_t)(buf_addr + buf_offset),
1395                         (uint32_t)buf_avail, 0);
1396
1397         mbuf_offset = 0;
1398         mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
1399         while (1) {
1400                 uint64_t hpa;
1401
1402                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
1403
1404                 /*
1405                  * A desc buf might across two host physical pages that are
1406                  * not continuous. In such case (gpa_to_hpa returns 0), data
1407                  * will be copied even though zero copy is enabled.
1408                  */
1409                 if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,
1410                                         buf_iova + buf_offset, cpy_len)))) {
1411                         cur->data_len = cpy_len;
1412                         cur->data_off = 0;
1413                         cur->buf_addr =
1414                                 (void *)(uintptr_t)(buf_addr + buf_offset);
1415                         cur->buf_iova = hpa;
1416
1417                         /*
1418                          * In zero copy mode, one mbuf can only reference data
1419                          * for one or partial of one desc buff.
1420                          */
1421                         mbuf_avail = cpy_len;
1422                 } else {
1423                         if (likely(cpy_len > MAX_BATCH_LEN ||
1424                                    vq->batch_copy_nb_elems >= vq->size ||
1425                                    (hdr && cur == m))) {
1426                                 rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
1427                                                                    mbuf_offset),
1428                                            (void *)((uintptr_t)(buf_addr +
1429                                                            buf_offset)),
1430                                            cpy_len);
1431                         } else {
1432                                 batch_copy[vq->batch_copy_nb_elems].dst =
1433                                         rte_pktmbuf_mtod_offset(cur, void *,
1434                                                                 mbuf_offset);
1435                                 batch_copy[vq->batch_copy_nb_elems].src =
1436                                         (void *)((uintptr_t)(buf_addr +
1437                                                                 buf_offset));
1438                                 batch_copy[vq->batch_copy_nb_elems].len =
1439                                         cpy_len;
1440                                 vq->batch_copy_nb_elems++;
1441                         }
1442                 }
1443
1444                 mbuf_avail  -= cpy_len;
1445                 mbuf_offset += cpy_len;
1446                 buf_avail -= cpy_len;
1447                 buf_offset += cpy_len;
1448
1449                 /* This buf reaches to its end, get the next one */
1450                 if (buf_avail == 0) {
1451                         if (++vec_idx >= nr_vec)
1452                                 break;
1453
1454                         buf_addr = buf_vec[vec_idx].buf_addr;
1455                         buf_iova = buf_vec[vec_idx].buf_iova;
1456                         buf_len = buf_vec[vec_idx].buf_len;
1457
1458                         buf_offset = 0;
1459                         buf_avail  = buf_len;
1460
1461                         PRINT_PACKET(dev, (uintptr_t)buf_addr,
1462                                         (uint32_t)buf_avail, 0);
1463                 }
1464
1465                 /*
1466                  * This mbuf reaches to its end, get a new one
1467                  * to hold more data.
1468                  */
1469                 if (mbuf_avail == 0) {
1470                         cur = rte_pktmbuf_alloc(mbuf_pool);
1471                         if (unlikely(cur == NULL)) {
1472                                 RTE_LOG(ERR, VHOST_DATA, "Failed to "
1473                                         "allocate memory for mbuf.\n");
1474                                 error = -1;
1475                                 goto out;
1476                         }
1477                         if (unlikely(dev->dequeue_zero_copy))
1478                                 rte_mbuf_refcnt_update(cur, 1);
1479
1480                         prev->next = cur;
1481                         prev->data_len = mbuf_offset;
1482                         m->nb_segs += 1;
1483                         m->pkt_len += mbuf_offset;
1484                         prev = cur;
1485
1486                         mbuf_offset = 0;
1487                         mbuf_avail  = cur->buf_len - RTE_PKTMBUF_HEADROOM;
1488                 }
1489         }
1490
1491         prev->data_len = mbuf_offset;
1492         m->pkt_len    += mbuf_offset;
1493
1494         if (hdr)
1495                 vhost_dequeue_offload(hdr, m);
1496
1497 out:
1498
1499         return error;
1500 }
1501
1502 static __rte_always_inline struct zcopy_mbuf *
1503 get_zmbuf(struct vhost_virtqueue *vq)
1504 {
1505         uint16_t i;
1506         uint16_t last;
1507         int tries = 0;
1508
1509         /* search [last_zmbuf_idx, zmbuf_size) */
1510         i = vq->last_zmbuf_idx;
1511         last = vq->zmbuf_size;
1512
1513 again:
1514         for (; i < last; i++) {
1515                 if (vq->zmbufs[i].in_use == 0) {
1516                         vq->last_zmbuf_idx = i + 1;
1517                         vq->zmbufs[i].in_use = 1;
1518                         return &vq->zmbufs[i];
1519                 }
1520         }
1521
1522         tries++;
1523         if (tries == 1) {
1524                 /* search [0, last_zmbuf_idx) */
1525                 i = 0;
1526                 last = vq->last_zmbuf_idx;
1527                 goto again;
1528         }
1529
1530         return NULL;
1531 }
1532
1533 static void
1534 virtio_dev_extbuf_free(void *addr __rte_unused, void *opaque)
1535 {
1536         rte_free(opaque);
1537 }
1538
1539 static int
1540 virtio_dev_extbuf_alloc(struct rte_mbuf *pkt, uint32_t size)
1541 {
1542         struct rte_mbuf_ext_shared_info *shinfo = NULL;
1543         uint32_t total_len = RTE_PKTMBUF_HEADROOM + size;
1544         uint16_t buf_len;
1545         rte_iova_t iova;
1546         void *buf;
1547
1548         /* Try to use pkt buffer to store shinfo to reduce the amount of memory
1549          * required, otherwise store shinfo in the new buffer.
1550          */
1551         if (rte_pktmbuf_tailroom(pkt) >= sizeof(*shinfo))
1552                 shinfo = rte_pktmbuf_mtod(pkt,
1553                                           struct rte_mbuf_ext_shared_info *);
1554         else {
1555                 total_len += sizeof(*shinfo) + sizeof(uintptr_t);
1556                 total_len = RTE_ALIGN_CEIL(total_len, sizeof(uintptr_t));
1557         }
1558
1559         if (unlikely(total_len > UINT16_MAX))
1560                 return -ENOSPC;
1561
1562         buf_len = total_len;
1563         buf = rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE);
1564         if (unlikely(buf == NULL))
1565                 return -ENOMEM;
1566
1567         /* Initialize shinfo */
1568         if (shinfo) {
1569                 shinfo->free_cb = virtio_dev_extbuf_free;
1570                 shinfo->fcb_opaque = buf;
1571                 rte_mbuf_ext_refcnt_set(shinfo, 1);
1572         } else {
1573                 shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len,
1574                                               virtio_dev_extbuf_free, buf);
1575                 if (unlikely(shinfo == NULL)) {
1576                         rte_free(buf);
1577                         RTE_LOG(ERR, VHOST_DATA, "Failed to init shinfo\n");
1578                         return -1;
1579                 }
1580         }
1581
1582         iova = rte_malloc_virt2iova(buf);
1583         rte_pktmbuf_attach_extbuf(pkt, buf, iova, buf_len, shinfo);
1584         rte_pktmbuf_reset_headroom(pkt);
1585
1586         return 0;
1587 }
1588
1589 /*
1590  * Allocate a host supported pktmbuf.
1591  */
1592 static __rte_always_inline struct rte_mbuf *
1593 virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct rte_mempool *mp,
1594                          uint32_t data_len)
1595 {
1596         struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp);
1597
1598         if (unlikely(pkt == NULL))
1599                 return NULL;
1600
1601         if (rte_pktmbuf_tailroom(pkt) >= data_len)
1602                 return pkt;
1603
1604         /* attach an external buffer if supported */
1605         if (dev->extbuf && !virtio_dev_extbuf_alloc(pkt, data_len))
1606                 return pkt;
1607
1608         /* check if chained buffers are allowed */
1609         if (!dev->linearbuf)
1610                 return pkt;
1611
1612         /* Data doesn't fit into the buffer and the host supports
1613          * only linear buffers
1614          */
1615         rte_pktmbuf_free(pkt);
1616
1617         return NULL;
1618 }
1619
1620 static __rte_noinline uint16_t
1621 virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
1622         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1623 {
1624         uint16_t i;
1625         uint16_t free_entries;
1626
1627         if (unlikely(dev->dequeue_zero_copy)) {
1628                 struct zcopy_mbuf *zmbuf, *next;
1629
1630                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1631                      zmbuf != NULL; zmbuf = next) {
1632                         next = TAILQ_NEXT(zmbuf, next);
1633
1634                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1635                                 update_shadow_used_ring_split(vq,
1636                                                 zmbuf->desc_idx, 0);
1637                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1638                                 restore_mbuf(zmbuf->mbuf);
1639                                 rte_pktmbuf_free(zmbuf->mbuf);
1640                                 put_zmbuf(zmbuf);
1641                                 vq->nr_zmbuf -= 1;
1642                         }
1643                 }
1644
1645                 if (likely(vq->shadow_used_idx)) {
1646                         flush_shadow_used_ring_split(dev, vq);
1647                         vhost_vring_call_split(dev, vq);
1648                 }
1649         }
1650
1651         free_entries = *((volatile uint16_t *)&vq->avail->idx) -
1652                         vq->last_avail_idx;
1653         if (free_entries == 0)
1654                 return 0;
1655
1656         /*
1657          * The ordering between avail index and
1658          * desc reads needs to be enforced.
1659          */
1660         rte_smp_rmb();
1661
1662         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
1663
1664         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1665
1666         count = RTE_MIN(count, MAX_PKT_BURST);
1667         count = RTE_MIN(count, free_entries);
1668         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
1669                         dev->vid, count);
1670
1671         for (i = 0; i < count; i++) {
1672                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
1673                 uint16_t head_idx;
1674                 uint32_t buf_len;
1675                 uint16_t nr_vec = 0;
1676                 int err;
1677
1678                 if (unlikely(fill_vec_buf_split(dev, vq,
1679                                                 vq->last_avail_idx + i,
1680                                                 &nr_vec, buf_vec,
1681                                                 &head_idx, &buf_len,
1682                                                 VHOST_ACCESS_RO) < 0))
1683                         break;
1684
1685                 if (likely(dev->dequeue_zero_copy == 0))
1686                         update_shadow_used_ring_split(vq, head_idx, 0);
1687
1688                 pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
1689                 if (unlikely(pkts[i] == NULL))
1690                         break;
1691
1692                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
1693                                 mbuf_pool);
1694                 if (unlikely(err)) {
1695                         rte_pktmbuf_free(pkts[i]);
1696                         break;
1697                 }
1698
1699                 if (unlikely(dev->dequeue_zero_copy)) {
1700                         struct zcopy_mbuf *zmbuf;
1701
1702                         zmbuf = get_zmbuf(vq);
1703                         if (!zmbuf) {
1704                                 rte_pktmbuf_free(pkts[i]);
1705                                 break;
1706                         }
1707                         zmbuf->mbuf = pkts[i];
1708                         zmbuf->desc_idx = head_idx;
1709
1710                         /*
1711                          * Pin lock the mbuf; we will check later to see
1712                          * whether the mbuf is freed (when we are the last
1713                          * user) or not. If that's the case, we then could
1714                          * update the used ring safely.
1715                          */
1716                         rte_mbuf_refcnt_update(pkts[i], 1);
1717
1718                         vq->nr_zmbuf += 1;
1719                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1720                 }
1721         }
1722         vq->last_avail_idx += i;
1723
1724         if (likely(dev->dequeue_zero_copy == 0)) {
1725                 do_data_copy_dequeue(vq);
1726                 if (unlikely(i < count))
1727                         vq->shadow_used_idx = i;
1728                 if (likely(vq->shadow_used_idx)) {
1729                         flush_shadow_used_ring_split(dev, vq);
1730                         vhost_vring_call_split(dev, vq);
1731                 }
1732         }
1733
1734         return i;
1735 }
1736
1737 static __rte_always_inline int
1738 vhost_reserve_avail_batch_packed(struct virtio_net *dev,
1739                                  struct vhost_virtqueue *vq,
1740                                  struct rte_mempool *mbuf_pool,
1741                                  struct rte_mbuf **pkts,
1742                                  uint16_t avail_idx,
1743                                  uintptr_t *desc_addrs,
1744                                  uint16_t *ids)
1745 {
1746         bool wrap = vq->avail_wrap_counter;
1747         struct vring_packed_desc *descs = vq->desc_packed;
1748         struct virtio_net_hdr *hdr;
1749         uint64_t lens[PACKED_BATCH_SIZE];
1750         uint64_t buf_lens[PACKED_BATCH_SIZE];
1751         uint32_t buf_offset = dev->vhost_hlen;
1752         uint16_t flags, i;
1753
1754         if (unlikely(avail_idx & PACKED_BATCH_MASK))
1755                 return -1;
1756         if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))
1757                 return -1;
1758
1759         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1760                 flags = descs[avail_idx + i].flags;
1761                 if (unlikely((wrap != !!(flags & VRING_DESC_F_AVAIL)) ||
1762                              (wrap == !!(flags & VRING_DESC_F_USED))  ||
1763                              (flags & PACKED_DESC_SINGLE_DEQUEUE_FLAG)))
1764                         return -1;
1765         }
1766
1767         rte_smp_rmb();
1768
1769         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1770                 lens[i] = descs[avail_idx + i].len;
1771
1772         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1773                 desc_addrs[i] = vhost_iova_to_vva(dev, vq,
1774                                                   descs[avail_idx + i].addr,
1775                                                   &lens[i], VHOST_ACCESS_RW);
1776         }
1777
1778         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1779                 if (unlikely((lens[i] != descs[avail_idx + i].len)))
1780                         return -1;
1781         }
1782
1783         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1784                 pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, lens[i]);
1785                 if (!pkts[i])
1786                         goto free_buf;
1787         }
1788
1789         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1790                 buf_lens[i] = pkts[i]->buf_len - pkts[i]->data_off;
1791
1792         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1793                 if (unlikely(buf_lens[i] < (lens[i] - buf_offset)))
1794                         goto free_buf;
1795         }
1796
1797         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1798                 pkts[i]->pkt_len = descs[avail_idx + i].len - buf_offset;
1799                 pkts[i]->data_len = pkts[i]->pkt_len;
1800                 ids[i] = descs[avail_idx + i].id;
1801         }
1802
1803         if (virtio_net_with_host_offload(dev)) {
1804                 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
1805                         hdr = (struct virtio_net_hdr *)(desc_addrs[i]);
1806                         vhost_dequeue_offload(hdr, pkts[i]);
1807                 }
1808         }
1809
1810         return 0;
1811
1812 free_buf:
1813         for (i = 0; i < PACKED_BATCH_SIZE; i++)
1814                 rte_pktmbuf_free(pkts[i]);
1815
1816         return -1;
1817 }
1818
1819 static __rte_unused int
1820 virtio_dev_tx_batch_packed(struct virtio_net *dev,
1821                            struct vhost_virtqueue *vq,
1822                            struct rte_mempool *mbuf_pool,
1823                            struct rte_mbuf **pkts)
1824 {
1825         uint16_t avail_idx = vq->last_avail_idx;
1826         uint32_t buf_offset = dev->vhost_hlen;
1827         uintptr_t desc_addrs[PACKED_BATCH_SIZE];
1828         uint16_t ids[PACKED_BATCH_SIZE];
1829         uint16_t i;
1830
1831         if (vhost_reserve_avail_batch_packed(dev, vq, mbuf_pool, pkts,
1832                                              avail_idx, desc_addrs, ids))
1833                 return -1;
1834
1835         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1836                 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]);
1837
1838         vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
1839                 rte_memcpy(rte_pktmbuf_mtod_offset(pkts[i], void *, 0),
1840                            (void *)(uintptr_t)(desc_addrs[i] + buf_offset),
1841                            pkts[i]->pkt_len);
1842
1843         vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
1844
1845         return 0;
1846 }
1847
1848 static __rte_always_inline int
1849 vhost_dequeue_single_packed(struct virtio_net *dev,
1850                             struct vhost_virtqueue *vq,
1851                             struct rte_mempool *mbuf_pool,
1852                             struct rte_mbuf **pkts,
1853                             uint16_t *buf_id,
1854                             uint16_t *desc_count)
1855 {
1856         struct buf_vector buf_vec[BUF_VECTOR_MAX];
1857         uint32_t buf_len;
1858         uint16_t nr_vec = 0;
1859         int err;
1860
1861         if (unlikely(fill_vec_buf_packed(dev, vq,
1862                                          vq->last_avail_idx, desc_count,
1863                                          buf_vec, &nr_vec,
1864                                          buf_id, &buf_len,
1865                                          VHOST_ACCESS_RO) < 0))
1866                 return -1;
1867
1868         *pkts = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
1869         if (unlikely(*pkts == NULL)) {
1870                 RTE_LOG(ERR, VHOST_DATA,
1871                         "Failed to allocate memory for mbuf.\n");
1872                 return -1;
1873         }
1874
1875         err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, *pkts,
1876                                 mbuf_pool);
1877         if (unlikely(err)) {
1878                 rte_pktmbuf_free(*pkts);
1879                 return -1;
1880         }
1881
1882         return 0;
1883 }
1884
1885 static __rte_unused int
1886 virtio_dev_tx_single_packed(struct virtio_net *dev,
1887                             struct vhost_virtqueue *vq,
1888                             struct rte_mempool *mbuf_pool,
1889                             struct rte_mbuf **pkts)
1890 {
1891
1892         uint16_t buf_id, desc_count;
1893
1894         if (vhost_dequeue_single_packed(dev, vq, mbuf_pool, pkts, &buf_id,
1895                                         &desc_count))
1896                 return -1;
1897
1898         vq_inc_last_avail_packed(vq, desc_count);
1899
1900         return 0;
1901 }
1902
1903 static __rte_noinline uint16_t
1904 virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
1905         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1906 {
1907         uint16_t i;
1908
1909         if (unlikely(dev->dequeue_zero_copy)) {
1910                 struct zcopy_mbuf *zmbuf, *next;
1911
1912                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1913                      zmbuf != NULL; zmbuf = next) {
1914                         next = TAILQ_NEXT(zmbuf, next);
1915
1916                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1917                                 update_shadow_used_ring_packed(vq,
1918                                                 zmbuf->desc_idx,
1919                                                 0,
1920                                                 zmbuf->desc_count);
1921
1922                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1923                                 restore_mbuf(zmbuf->mbuf);
1924                                 rte_pktmbuf_free(zmbuf->mbuf);
1925                                 put_zmbuf(zmbuf);
1926                                 vq->nr_zmbuf -= 1;
1927                         }
1928                 }
1929
1930                 if (likely(vq->shadow_used_idx)) {
1931                         flush_shadow_used_ring_packed(dev, vq);
1932                         vhost_vring_call_packed(dev, vq);
1933                 }
1934         }
1935
1936         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1937
1938         count = RTE_MIN(count, MAX_PKT_BURST);
1939         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
1940                         dev->vid, count);
1941
1942         for (i = 0; i < count; i++) {
1943                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
1944                 uint16_t buf_id;
1945                 uint32_t buf_len;
1946                 uint16_t desc_count, nr_vec = 0;
1947                 int err;
1948
1949                 if (unlikely(fill_vec_buf_packed(dev, vq,
1950                                                 vq->last_avail_idx, &desc_count,
1951                                                 buf_vec, &nr_vec,
1952                                                 &buf_id, &buf_len,
1953                                                 VHOST_ACCESS_RO) < 0))
1954                         break;
1955
1956                 if (likely(dev->dequeue_zero_copy == 0))
1957                         update_shadow_used_ring_packed(vq, buf_id, 0,
1958                                         desc_count);
1959
1960                 pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
1961                 if (unlikely(pkts[i] == NULL))
1962                         break;
1963
1964                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
1965                                 mbuf_pool);
1966                 if (unlikely(err)) {
1967                         rte_pktmbuf_free(pkts[i]);
1968                         break;
1969                 }
1970
1971                 if (unlikely(dev->dequeue_zero_copy)) {
1972                         struct zcopy_mbuf *zmbuf;
1973
1974                         zmbuf = get_zmbuf(vq);
1975                         if (!zmbuf) {
1976                                 rte_pktmbuf_free(pkts[i]);
1977                                 break;
1978                         }
1979                         zmbuf->mbuf = pkts[i];
1980                         zmbuf->desc_idx = buf_id;
1981                         zmbuf->desc_count = desc_count;
1982
1983                         /*
1984                          * Pin lock the mbuf; we will check later to see
1985                          * whether the mbuf is freed (when we are the last
1986                          * user) or not. If that's the case, we then could
1987                          * update the used ring safely.
1988                          */
1989                         rte_mbuf_refcnt_update(pkts[i], 1);
1990
1991                         vq->nr_zmbuf += 1;
1992                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1993                 }
1994
1995                 vq_inc_last_avail_packed(vq, desc_count);
1996         }
1997
1998         if (likely(dev->dequeue_zero_copy == 0)) {
1999                 do_data_copy_dequeue(vq);
2000                 if (unlikely(i < count))
2001                         vq->shadow_used_idx = i;
2002                 if (likely(vq->shadow_used_idx)) {
2003                         flush_shadow_used_ring_packed(dev, vq);
2004                         vhost_vring_call_packed(dev, vq);
2005                 }
2006         }
2007
2008         return i;
2009 }
2010
2011 uint16_t
2012 rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
2013         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
2014 {
2015         struct virtio_net *dev;
2016         struct rte_mbuf *rarp_mbuf = NULL;
2017         struct vhost_virtqueue *vq;
2018
2019         dev = get_device(vid);
2020         if (!dev)
2021                 return 0;
2022
2023         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
2024                 RTE_LOG(ERR, VHOST_DATA,
2025                         "(%d) %s: built-in vhost net backend is disabled.\n",
2026                         dev->vid, __func__);
2027                 return 0;
2028         }
2029
2030         if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
2031                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
2032                         dev->vid, __func__, queue_id);
2033                 return 0;
2034         }
2035
2036         vq = dev->virtqueue[queue_id];
2037
2038         if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
2039                 return 0;
2040
2041         if (unlikely(vq->enabled == 0)) {
2042                 count = 0;
2043                 goto out_access_unlock;
2044         }
2045
2046         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
2047                 vhost_user_iotlb_rd_lock(vq);
2048
2049         if (unlikely(vq->access_ok == 0))
2050                 if (unlikely(vring_translate(dev, vq) < 0)) {
2051                         count = 0;
2052                         goto out;
2053                 }
2054
2055         /*
2056          * Construct a RARP broadcast packet, and inject it to the "pkts"
2057          * array, to looks like that guest actually send such packet.
2058          *
2059          * Check user_send_rarp() for more information.
2060          *
2061          * broadcast_rarp shares a cacheline in the virtio_net structure
2062          * with some fields that are accessed during enqueue and
2063          * rte_atomic16_cmpset() causes a write if using cmpxchg. This could
2064          * result in false sharing between enqueue and dequeue.
2065          *
2066          * Prevent unnecessary false sharing by reading broadcast_rarp first
2067          * and only performing cmpset if the read indicates it is likely to
2068          * be set.
2069          */
2070         if (unlikely(rte_atomic16_read(&dev->broadcast_rarp) &&
2071                         rte_atomic16_cmpset((volatile uint16_t *)
2072                                 &dev->broadcast_rarp.cnt, 1, 0))) {
2073
2074                 rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
2075                 if (rarp_mbuf == NULL) {
2076                         RTE_LOG(ERR, VHOST_DATA,
2077                                 "Failed to make RARP packet.\n");
2078                         count = 0;
2079                         goto out;
2080                 }
2081                 count -= 1;
2082         }
2083
2084         if (vq_is_packed(dev))
2085                 count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count);
2086         else
2087                 count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);
2088
2089 out:
2090         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
2091                 vhost_user_iotlb_rd_unlock(vq);
2092
2093 out_access_unlock:
2094         rte_spinlock_unlock(&vq->access_lock);
2095
2096         if (unlikely(rarp_mbuf != NULL)) {
2097                 /*
2098                  * Inject it to the head of "pkts" array, so that switch's mac
2099                  * learning table will get updated first.
2100                  */
2101                 memmove(&pkts[1], pkts, count * sizeof(struct rte_mbuf *));
2102                 pkts[0] = rarp_mbuf;
2103                 count += 1;
2104         }
2105
2106         return count;
2107 }