lib/vhost/vhost.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <linux/vhost.h>
   6 #include <linux/virtio_net.h>
   7 #include <stddef.h>
   8 #include <stdint.h>
   9 #include <stdlib.h>
  10 #ifdef RTE_LIBRTE_VHOST_NUMA
  11 #include <numa.h>
  12 #include <numaif.h>
  13 #endif
  14
  15 #include <rte_errno.h>
  16 #include <rte_ethdev.h>
  17 #include <rte_log.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_memory.h>
  20 #include <rte_malloc.h>
  21 #include <rte_vhost.h>
  22 #include <rte_rwlock.h>
  23
  24 #include "iotlb.h"
  25 #include "vhost.h"
  26 #include "vhost_user.h"
  27
  28 struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
  29 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER;
  30
  31 /* Called with iotlb_lock read-locked */
  32 uint64_t
  33 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
  34                     uint64_t iova, uint64_t *size, uint8_t perm)
  35 {
  36         uint64_t vva, tmp_size;
  37
  38         if (unlikely(!*size))
  39                 return 0;
  40
  41         tmp_size = *size;
  42
  43         vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm);
  44         if (tmp_size == *size)
  45                 return vva;
  46
  47         iova += tmp_size;
  48
  49         if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) {
  50                 /*
  51                  * iotlb_lock is read-locked for a full burst,
  52                  * but it only protects the iotlb cache.
  53                  * In case of IOTLB miss, we might block on the socket,
  54                  * which could cause a deadlock with QEMU if an IOTLB update
  55                  * is being handled. We can safely unlock here to avoid it.
  56                  */
  57                 vhost_user_iotlb_rd_unlock(vq);
  58
  59                 vhost_user_iotlb_pending_insert(vq, iova, perm);
  60                 if (vhost_user_iotlb_miss(dev, iova, perm)) {
  61                         VHOST_LOG_CONFIG(ERR,
  62                                 "IOTLB miss req failed for IOVA 0x%" PRIx64 "\n",
  63                                 iova);
  64                         vhost_user_iotlb_pending_remove(vq, iova, 1, perm);
  65                 }
  66
  67                 vhost_user_iotlb_rd_lock(vq);
  68         }
  69
  70         return 0;
  71 }
  72
  73 #define VHOST_LOG_PAGE  4096
  74
  75 /*
  76  * Atomically set a bit in memory.
  77  */
  78 static __rte_always_inline void
  79 vhost_set_bit(unsigned int nr, volatile uint8_t *addr)
  80 {
  81 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
  82         /*
  83          * __sync_ built-ins are deprecated, but __atomic_ ones
  84          * are sub-optimized in older GCC versions.
  85          */
  86         __sync_fetch_and_or_1(addr, (1U << nr));
  87 #else
  88         __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED);
  89 #endif
  90 }
  91
  92 static __rte_always_inline void
  93 vhost_log_page(uint8_t *log_base, uint64_t page)
  94 {
  95         vhost_set_bit(page % 8, &log_base[page / 8]);
  96 }
  97
  98 void
  99 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
 100 {
 101         uint64_t page;
 102
 103         if (unlikely(!dev->log_base || !len))
 104                 return;
 105
 106         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 107                 return;
 108
 109         /* To make sure guest memory updates are committed before logging */
 110         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 111
 112         page = addr / VHOST_LOG_PAGE;
 113         while (page * VHOST_LOG_PAGE < addr + len) {
 114                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 115                 page += 1;
 116         }
 117 }
 118
 119 void
 120 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 121                              uint64_t iova, uint64_t len)
 122 {
 123         uint64_t hva, gpa, map_len;
 124         map_len = len;
 125
 126         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 127         if (map_len != len) {
 128                 VHOST_LOG_DATA(ERR,
 129                         "Failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 130                         iova);
 131                 return;
 132         }
 133
 134         gpa = hva_to_gpa(dev, hva, len);
 135         if (gpa)
 136                 __vhost_log_write(dev, gpa, len);
 137 }
 138
 139 void
 140 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
 141 {
 142         unsigned long *log_base;
 143         int i;
 144
 145         if (unlikely(!dev->log_base))
 146                 return;
 147
 148         /* No cache, nothing to sync */
 149         if (unlikely(!vq->log_cache))
 150                 return;
 151
 152         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 153
 154         log_base = (unsigned long *)(uintptr_t)dev->log_base;
 155
 156         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 157                 struct log_cache_entry *elem = vq->log_cache + i;
 158
 159 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
 160                 /*
 161                  * '__sync' builtins are deprecated, but '__atomic' ones
 162                  * are sub-optimized in older GCC versions.
 163                  */
 164                 __sync_fetch_and_or(log_base + elem->offset, elem->val);
 165 #else
 166                 __atomic_fetch_or(log_base + elem->offset, elem->val,
 167                                 __ATOMIC_RELAXED);
 168 #endif
 169         }
 170
 171         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 172
 173         vq->log_cache_nb_elem = 0;
 174 }
 175
 176 static __rte_always_inline void
 177 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
 178                         uint64_t page)
 179 {
 180         uint32_t bit_nr = page % (sizeof(unsigned long) << 3);
 181         uint32_t offset = page / (sizeof(unsigned long) << 3);
 182         int i;
 183
 184         if (unlikely(!vq->log_cache)) {
 185                 /* No logging cache allocated, write dirty log map directly */
 186                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 187                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 188
 189                 return;
 190         }
 191
 192         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 193                 struct log_cache_entry *elem = vq->log_cache + i;
 194
 195                 if (elem->offset == offset) {
 196                         elem->val |= (1UL << bit_nr);
 197                         return;
 198                 }
 199         }
 200
 201         if (unlikely(i >= VHOST_LOG_CACHE_NR)) {
 202                 /*
 203                  * No more room for a new log cache entry,
 204                  * so write the dirty log map directly.
 205                  */
 206                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 207                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 208
 209                 return;
 210         }
 211
 212         vq->log_cache[i].offset = offset;
 213         vq->log_cache[i].val = (1UL << bit_nr);
 214         vq->log_cache_nb_elem++;
 215 }
 216
 217 void
 218 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
 219                         uint64_t addr, uint64_t len)
 220 {
 221         uint64_t page;
 222
 223         if (unlikely(!dev->log_base || !len))
 224                 return;
 225
 226         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 227                 return;
 228
 229         page = addr / VHOST_LOG_PAGE;
 230         while (page * VHOST_LOG_PAGE < addr + len) {
 231                 vhost_log_cache_page(dev, vq, page);
 232                 page += 1;
 233         }
 234 }
 235
 236 void
 237 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 238                              uint64_t iova, uint64_t len)
 239 {
 240         uint64_t hva, gpa, map_len;
 241         map_len = len;
 242
 243         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 244         if (map_len != len) {
 245                 VHOST_LOG_DATA(ERR,
 246                         "Failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 247                         iova);
 248                 return;
 249         }
 250
 251         gpa = hva_to_gpa(dev, hva, len);
 252         if (gpa)
 253                 __vhost_log_cache_write(dev, vq, gpa, len);
 254 }
 255
 256 void *
 257 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
 258                 uint64_t desc_addr, uint64_t desc_len)
 259 {
 260         void *idesc;
 261         uint64_t src, dst;
 262         uint64_t len, remain = desc_len;
 263
 264         idesc = rte_malloc(__func__, desc_len, 0);
 265         if (unlikely(!idesc))
 266                 return NULL;
 267
 268         dst = (uint64_t)(uintptr_t)idesc;
 269
 270         while (remain) {
 271                 len = remain;
 272                 src = vhost_iova_to_vva(dev, vq, desc_addr, &len,
 273                                 VHOST_ACCESS_RO);
 274                 if (unlikely(!src || !len)) {
 275                         rte_free(idesc);
 276                         return NULL;
 277                 }
 278
 279                 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len);
 280
 281                 remain -= len;
 282                 dst += len;
 283                 desc_addr += len;
 284         }
 285
 286         return idesc;
 287 }
 288
 289 void
 290 cleanup_vq(struct vhost_virtqueue *vq, int destroy)
 291 {
 292         if ((vq->callfd >= 0) && (destroy != 0))
 293                 close(vq->callfd);
 294         if (vq->kickfd >= 0)
 295                 close(vq->kickfd);
 296 }
 297
 298 void
 299 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq)
 300 {
 301         if (!(dev->protocol_features &
 302             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
 303                 return;
 304
 305         if (vq_is_packed(dev)) {
 306                 if (vq->inflight_packed)
 307                         vq->inflight_packed = NULL;
 308         } else {
 309                 if (vq->inflight_split)
 310                         vq->inflight_split = NULL;
 311         }
 312
 313         if (vq->resubmit_inflight) {
 314                 if (vq->resubmit_inflight->resubmit_list) {
 315                         free(vq->resubmit_inflight->resubmit_list);
 316                         vq->resubmit_inflight->resubmit_list = NULL;
 317                 }
 318                 free(vq->resubmit_inflight);
 319                 vq->resubmit_inflight = NULL;
 320         }
 321 }
 322
 323 /*
 324  * Unmap any memory, close any file descriptors and
 325  * free any memory owned by a device.
 326  */
 327 void
 328 cleanup_device(struct virtio_net *dev, int destroy)
 329 {
 330         uint32_t i;
 331
 332         vhost_backend_cleanup(dev);
 333
 334         for (i = 0; i < dev->nr_vring; i++) {
 335                 cleanup_vq(dev->virtqueue[i], destroy);
 336                 cleanup_vq_inflight(dev, dev->virtqueue[i]);
 337         }
 338 }
 339
 340 static void
 341 vhost_free_async_mem(struct vhost_virtqueue *vq)
 342 {
 343         rte_free(vq->async_pkts_info);
 344
 345         rte_free(vq->async_buffers_packed);
 346         vq->async_buffers_packed = NULL;
 347         rte_free(vq->async_descs_split);
 348         vq->async_descs_split = NULL;
 349
 350         rte_free(vq->it_pool);
 351         rte_free(vq->vec_pool);
 352
 353         vq->async_pkts_info = NULL;
 354         vq->it_pool = NULL;
 355         vq->vec_pool = NULL;
 356 }
 357
 358 void
 359 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 360 {
 361         if (vq_is_packed(dev))
 362                 rte_free(vq->shadow_used_packed);
 363         else
 364                 rte_free(vq->shadow_used_split);
 365
 366         vhost_free_async_mem(vq);
 367         rte_free(vq->batch_copy_elems);
 368         if (vq->iotlb_pool)
 369                 rte_mempool_free(vq->iotlb_pool);
 370         rte_free(vq->log_cache);
 371         rte_free(vq);
 372 }
 373
 374 /*
 375  * Release virtqueues and device memory.
 376  */
 377 static void
 378 free_device(struct virtio_net *dev)
 379 {
 380         uint32_t i;
 381
 382         for (i = 0; i < dev->nr_vring; i++)
 383                 free_vq(dev, dev->virtqueue[i]);
 384
 385         rte_free(dev);
 386 }
 387
 388 static __rte_always_inline int
 389 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 390 {
 391         if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG))))
 392                 return 0;
 393
 394         vq->log_guest_addr = translate_log_addr(dev, vq,
 395                                                 vq->ring_addrs.log_guest_addr);
 396         if (vq->log_guest_addr == 0)
 397                 return -1;
 398
 399         return 0;
 400 }
 401
 402 /*
 403  * Converts vring log address to GPA
 404  * If IOMMU is enabled, the log address is IOVA
 405  * If IOMMU not enabled, the log address is already GPA
 406  *
 407  * Caller should have iotlb_lock read-locked
 408  */
 409 uint64_t
 410 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 411                 uint64_t log_addr)
 412 {
 413         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 414                 const uint64_t exp_size = sizeof(uint64_t);
 415                 uint64_t hva, gpa;
 416                 uint64_t size = exp_size;
 417
 418                 hva = vhost_iova_to_vva(dev, vq, log_addr,
 419                                         &size, VHOST_ACCESS_RW);
 420
 421                 if (size != exp_size)
 422                         return 0;
 423
 424                 gpa = hva_to_gpa(dev, hva, exp_size);
 425                 if (!gpa) {
 426                         VHOST_LOG_CONFIG(ERR,
 427                                 "VQ: Failed to find GPA for log_addr: 0x%"
 428                                 PRIx64 " hva: 0x%" PRIx64 "\n",
 429                                 log_addr, hva);
 430                         return 0;
 431                 }
 432                 return gpa;
 433
 434         } else
 435                 return log_addr;
 436 }
 437
 438 /* Caller should have iotlb_lock read-locked */
 439 static int
 440 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 441 {
 442         uint64_t req_size, size;
 443
 444         req_size = sizeof(struct vring_desc) * vq->size;
 445         size = req_size;
 446         vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 447                                                 vq->ring_addrs.desc_user_addr,
 448                                                 &size, VHOST_ACCESS_RW);
 449         if (!vq->desc || size != req_size)
 450                 return -1;
 451
 452         req_size = sizeof(struct vring_avail);
 453         req_size += sizeof(uint16_t) * vq->size;
 454         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 455                 req_size += sizeof(uint16_t);
 456         size = req_size;
 457         vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 458                                                 vq->ring_addrs.avail_user_addr,
 459                                                 &size, VHOST_ACCESS_RW);
 460         if (!vq->avail || size != req_size)
 461                 return -1;
 462
 463         req_size = sizeof(struct vring_used);
 464         req_size += sizeof(struct vring_used_elem) * vq->size;
 465         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 466                 req_size += sizeof(uint16_t);
 467         size = req_size;
 468         vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 469                                                 vq->ring_addrs.used_user_addr,
 470                                                 &size, VHOST_ACCESS_RW);
 471         if (!vq->used || size != req_size)
 472                 return -1;
 473
 474         return 0;
 475 }
 476
 477 /* Caller should have iotlb_lock read-locked */
 478 static int
 479 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)
 480 {
 481         uint64_t req_size, size;
 482
 483         req_size = sizeof(struct vring_packed_desc) * vq->size;
 484         size = req_size;
 485         vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 486                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr,
 487                                 &size, VHOST_ACCESS_RW);
 488         if (!vq->desc_packed || size != req_size)
 489                 return -1;
 490
 491         req_size = sizeof(struct vring_packed_desc_event);
 492         size = req_size;
 493         vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t)
 494                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr,
 495                                 &size, VHOST_ACCESS_RW);
 496         if (!vq->driver_event || size != req_size)
 497                 return -1;
 498
 499         req_size = sizeof(struct vring_packed_desc_event);
 500         size = req_size;
 501         vq->device_event = (struct vring_packed_desc_event *)(uintptr_t)
 502                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr,
 503                                 &size, VHOST_ACCESS_RW);
 504         if (!vq->device_event || size != req_size)
 505                 return -1;
 506
 507         return 0;
 508 }
 509
 510 int
 511 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 512 {
 513
 514         if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
 515                 return -1;
 516
 517         if (vq_is_packed(dev)) {
 518                 if (vring_translate_packed(dev, vq) < 0)
 519                         return -1;
 520         } else {
 521                 if (vring_translate_split(dev, vq) < 0)
 522                         return -1;
 523         }
 524
 525         if (log_translate(dev, vq) < 0)
 526                 return -1;
 527
 528         vq->access_ok = true;
 529
 530         return 0;
 531 }
 532
 533 void
 534 vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 535 {
 536         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 537                 vhost_user_iotlb_wr_lock(vq);
 538
 539         vq->access_ok = false;
 540         vq->desc = NULL;
 541         vq->avail = NULL;
 542         vq->used = NULL;
 543         vq->log_guest_addr = 0;
 544
 545         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 546                 vhost_user_iotlb_wr_unlock(vq);
 547 }
 548
 549 static void
 550 init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 551 {
 552         struct vhost_virtqueue *vq;
 553
 554         if (vring_idx >= VHOST_MAX_VRING) {
 555                 VHOST_LOG_CONFIG(ERR,
 556                                 "Failed not init vring, out of bound (%d)\n",
 557                                 vring_idx);
 558                 return;
 559         }
 560
 561         vq = dev->virtqueue[vring_idx];
 562         if (!vq) {
 563                 VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n",
 564                                 vring_idx);
 565                 return;
 566         }
 567
 568         memset(vq, 0, sizeof(struct vhost_virtqueue));
 569
 570         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
 571         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
 572         vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
 573 }
 574
 575 static void
 576 reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 577 {
 578         struct vhost_virtqueue *vq;
 579         int callfd;
 580
 581         if (vring_idx >= VHOST_MAX_VRING) {
 582                 VHOST_LOG_CONFIG(ERR,
 583                                 "Failed not init vring, out of bound (%d)\n",
 584                                 vring_idx);
 585                 return;
 586         }
 587
 588         vq = dev->virtqueue[vring_idx];
 589         if (!vq) {
 590                 VHOST_LOG_CONFIG(ERR, "Virtqueue not allocated (%d)\n",
 591                                 vring_idx);
 592                 return;
 593         }
 594
 595         callfd = vq->callfd;
 596         init_vring_queue(dev, vring_idx);
 597         vq->callfd = callfd;
 598 }
 599
 600 int
 601 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 602 {
 603         struct vhost_virtqueue *vq;
 604         uint32_t i;
 605
 606         /* Also allocate holes, if any, up to requested vring index. */
 607         for (i = 0; i <= vring_idx; i++) {
 608                 if (dev->virtqueue[i])
 609                         continue;
 610
 611                 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0);
 612                 if (vq == NULL) {
 613                         VHOST_LOG_CONFIG(ERR,
 614                                 "Failed to allocate memory for vring:%u.\n", i);
 615                         return -1;
 616                 }
 617
 618                 dev->virtqueue[i] = vq;
 619                 init_vring_queue(dev, i);
 620                 rte_spinlock_init(&vq->access_lock);
 621                 vq->avail_wrap_counter = 1;
 622                 vq->used_wrap_counter = 1;
 623                 vq->signalled_used_valid = false;
 624         }
 625
 626         dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1);
 627
 628         return 0;
 629 }
 630
 631 /*
 632  * Reset some variables in device structure, while keeping few
 633  * others untouched, such as vid, ifname, nr_vring: they
 634  * should be same unless the device is removed.
 635  */
 636 void
 637 reset_device(struct virtio_net *dev)
 638 {
 639         uint32_t i;
 640
 641         dev->features = 0;
 642         dev->protocol_features = 0;
 643         dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 644
 645         for (i = 0; i < dev->nr_vring; i++)
 646                 reset_vring_queue(dev, i);
 647 }
 648
 649 /*
 650  * Invoked when there is a new vhost-user connection established (when
 651  * there is a new virtio device being attached).
 652  */
 653 int
 654 vhost_new_device(void)
 655 {
 656         struct virtio_net *dev;
 657         int i;
 658
 659         pthread_mutex_lock(&vhost_dev_lock);
 660         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
 661                 if (vhost_devices[i] == NULL)
 662                         break;
 663         }
 664
 665         if (i == MAX_VHOST_DEVICE) {
 666                 VHOST_LOG_CONFIG(ERR,
 667                         "Failed to find a free slot for new device.\n");
 668                 pthread_mutex_unlock(&vhost_dev_lock);
 669                 return -1;
 670         }
 671
 672         dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0);
 673         if (dev == NULL) {
 674                 VHOST_LOG_CONFIG(ERR,
 675                         "Failed to allocate memory for new dev.\n");
 676                 pthread_mutex_unlock(&vhost_dev_lock);
 677                 return -1;
 678         }
 679
 680         vhost_devices[i] = dev;
 681         pthread_mutex_unlock(&vhost_dev_lock);
 682
 683         dev->vid = i;
 684         dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 685         dev->slave_req_fd = -1;
 686         dev->postcopy_ufd = -1;
 687         rte_spinlock_init(&dev->slave_req_lock);
 688
 689         return i;
 690 }
 691
 692 void
 693 vhost_destroy_device_notify(struct virtio_net *dev)
 694 {
 695         struct rte_vdpa_device *vdpa_dev;
 696
 697         if (dev->flags & VIRTIO_DEV_RUNNING) {
 698                 vdpa_dev = dev->vdpa_dev;
 699                 if (vdpa_dev)
 700                         vdpa_dev->ops->dev_close(dev->vid);
 701                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 702                 dev->notify_ops->destroy_device(dev->vid);
 703         }
 704 }
 705
 706 /*
 707  * Invoked when there is the vhost-user connection is broken (when
 708  * the virtio device is being detached).
 709  */
 710 void
 711 vhost_destroy_device(int vid)
 712 {
 713         struct virtio_net *dev = get_device(vid);
 714
 715         if (dev == NULL)
 716                 return;
 717
 718         vhost_destroy_device_notify(dev);
 719
 720         cleanup_device(dev, 1);
 721         free_device(dev);
 722
 723         vhost_devices[vid] = NULL;
 724 }
 725
 726 void
 727 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev)
 728 {
 729         struct virtio_net *dev = get_device(vid);
 730
 731         if (dev == NULL)
 732                 return;
 733
 734         dev->vdpa_dev = vdpa_dev;
 735 }
 736
 737 void
 738 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len)
 739 {
 740         struct virtio_net *dev;
 741         unsigned int len;
 742
 743         dev = get_device(vid);
 744         if (dev == NULL)
 745                 return;
 746
 747         len = if_len > sizeof(dev->ifname) ?
 748                 sizeof(dev->ifname) : if_len;
 749
 750         strncpy(dev->ifname, if_name, len);
 751         dev->ifname[sizeof(dev->ifname) - 1] = '\0';
 752 }
 753
 754 void
 755 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags)
 756 {
 757         struct virtio_net *dev = get_device(vid);
 758
 759         if (dev == NULL)
 760                 return;
 761
 762         if (enable)
 763                 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 764         else
 765                 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 766         if (!compliant_ol_flags)
 767                 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS;
 768         else
 769                 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS;
 770 }
 771
 772 void
 773 vhost_enable_extbuf(int vid)
 774 {
 775         struct virtio_net *dev = get_device(vid);
 776
 777         if (dev == NULL)
 778                 return;
 779
 780         dev->extbuf = 1;
 781 }
 782
 783 void
 784 vhost_enable_linearbuf(int vid)
 785 {
 786         struct virtio_net *dev = get_device(vid);
 787
 788         if (dev == NULL)
 789                 return;
 790
 791         dev->linearbuf = 1;
 792 }
 793
 794 int
 795 rte_vhost_get_mtu(int vid, uint16_t *mtu)
 796 {
 797         struct virtio_net *dev = get_device(vid);
 798
 799         if (dev == NULL || mtu == NULL)
 800                 return -ENODEV;
 801
 802         if (!(dev->flags & VIRTIO_DEV_READY))
 803                 return -EAGAIN;
 804
 805         if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU)))
 806                 return -ENOTSUP;
 807
 808         *mtu = dev->mtu;
 809
 810         return 0;
 811 }
 812
 813 int
 814 rte_vhost_get_numa_node(int vid)
 815 {
 816 #ifdef RTE_LIBRTE_VHOST_NUMA
 817         struct virtio_net *dev = get_device(vid);
 818         int numa_node;
 819         int ret;
 820
 821         if (dev == NULL || numa_available() != 0)
 822                 return -1;
 823
 824         ret = get_mempolicy(&numa_node, NULL, 0, dev,
 825                             MPOL_F_NODE | MPOL_F_ADDR);
 826         if (ret < 0) {
 827                 VHOST_LOG_CONFIG(ERR,
 828                         "(%d) failed to query numa node: %s\n",
 829                         vid, rte_strerror(errno));
 830                 return -1;
 831         }
 832
 833         return numa_node;
 834 #else
 835         RTE_SET_USED(vid);
 836         return -1;
 837 #endif
 838 }
 839
 840 uint32_t
 841 rte_vhost_get_queue_num(int vid)
 842 {
 843         struct virtio_net *dev = get_device(vid);
 844
 845         if (dev == NULL)
 846                 return 0;
 847
 848         return dev->nr_vring / 2;
 849 }
 850
 851 uint16_t
 852 rte_vhost_get_vring_num(int vid)
 853 {
 854         struct virtio_net *dev = get_device(vid);
 855
 856         if (dev == NULL)
 857                 return 0;
 858
 859         return dev->nr_vring;
 860 }
 861
 862 int
 863 rte_vhost_get_ifname(int vid, char *buf, size_t len)
 864 {
 865         struct virtio_net *dev = get_device(vid);
 866
 867         if (dev == NULL || buf == NULL)
 868                 return -1;
 869
 870         len = RTE_MIN(len, sizeof(dev->ifname));
 871
 872         strncpy(buf, dev->ifname, len);
 873         buf[len - 1] = '\0';
 874
 875         return 0;
 876 }
 877
 878 int
 879 rte_vhost_get_negotiated_features(int vid, uint64_t *features)
 880 {
 881         struct virtio_net *dev;
 882
 883         dev = get_device(vid);
 884         if (dev == NULL || features == NULL)
 885                 return -1;
 886
 887         *features = dev->features;
 888         return 0;
 889 }
 890
 891 int
 892 rte_vhost_get_negotiated_protocol_features(int vid,
 893                                            uint64_t *protocol_features)
 894 {
 895         struct virtio_net *dev;
 896
 897         dev = get_device(vid);
 898         if (dev == NULL || protocol_features == NULL)
 899                 return -1;
 900
 901         *protocol_features = dev->protocol_features;
 902         return 0;
 903 }
 904
 905 int
 906 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem)
 907 {
 908         struct virtio_net *dev;
 909         struct rte_vhost_memory *m;
 910         size_t size;
 911
 912         dev = get_device(vid);
 913         if (dev == NULL || mem == NULL)
 914                 return -1;
 915
 916         size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region);
 917         m = malloc(sizeof(struct rte_vhost_memory) + size);
 918         if (!m)
 919                 return -1;
 920
 921         m->nregions = dev->mem->nregions;
 922         memcpy(m->regions, dev->mem->regions, size);
 923         *mem = m;
 924
 925         return 0;
 926 }
 927
 928 int
 929 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
 930                           struct rte_vhost_vring *vring)
 931 {
 932         struct virtio_net *dev;
 933         struct vhost_virtqueue *vq;
 934
 935         dev = get_device(vid);
 936         if (dev == NULL || vring == NULL)
 937                 return -1;
 938
 939         if (vring_idx >= VHOST_MAX_VRING)
 940                 return -1;
 941
 942         vq = dev->virtqueue[vring_idx];
 943         if (!vq)
 944                 return -1;
 945
 946         if (vq_is_packed(dev)) {
 947                 vring->desc_packed = vq->desc_packed;
 948                 vring->driver_event = vq->driver_event;
 949                 vring->device_event = vq->device_event;
 950         } else {
 951                 vring->desc = vq->desc;
 952                 vring->avail = vq->avail;
 953                 vring->used = vq->used;
 954         }
 955         vring->log_guest_addr  = vq->log_guest_addr;
 956
 957         vring->callfd  = vq->callfd;
 958         vring->kickfd  = vq->kickfd;
 959         vring->size    = vq->size;
 960
 961         return 0;
 962 }
 963
 964 int
 965 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx,
 966                                   struct rte_vhost_ring_inflight *vring)
 967 {
 968         struct virtio_net *dev;
 969         struct vhost_virtqueue *vq;
 970
 971         dev = get_device(vid);
 972         if (unlikely(!dev))
 973                 return -1;
 974
 975         if (vring_idx >= VHOST_MAX_VRING)
 976                 return -1;
 977
 978         vq = dev->virtqueue[vring_idx];
 979         if (unlikely(!vq))
 980                 return -1;
 981
 982         if (vq_is_packed(dev)) {
 983                 if (unlikely(!vq->inflight_packed))
 984                         return -1;
 985
 986                 vring->inflight_packed = vq->inflight_packed;
 987         } else {
 988                 if (unlikely(!vq->inflight_split))
 989                         return -1;
 990
 991                 vring->inflight_split = vq->inflight_split;
 992         }
 993
 994         vring->resubmit_inflight = vq->resubmit_inflight;
 995
 996         return 0;
 997 }
 998
 999 int
1000 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx,
1001                                   uint16_t idx)
1002 {
1003         struct vhost_virtqueue *vq;
1004         struct virtio_net *dev;
1005
1006         dev = get_device(vid);
1007         if (unlikely(!dev))
1008                 return -1;
1009
1010         if (unlikely(!(dev->protocol_features &
1011             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1012                 return 0;
1013
1014         if (unlikely(vq_is_packed(dev)))
1015                 return -1;
1016
1017         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1018                 return -1;
1019
1020         vq = dev->virtqueue[vring_idx];
1021         if (unlikely(!vq))
1022                 return -1;
1023
1024         if (unlikely(!vq->inflight_split))
1025                 return -1;
1026
1027         if (unlikely(idx >= vq->size))
1028                 return -1;
1029
1030         vq->inflight_split->desc[idx].counter = vq->global_counter++;
1031         vq->inflight_split->desc[idx].inflight = 1;
1032         return 0;
1033 }
1034
1035 int
1036 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx,
1037                                    uint16_t head, uint16_t last,
1038                                    uint16_t *inflight_entry)
1039 {
1040         struct rte_vhost_inflight_info_packed *inflight_info;
1041         struct virtio_net *dev;
1042         struct vhost_virtqueue *vq;
1043         struct vring_packed_desc *desc;
1044         uint16_t old_free_head, free_head;
1045
1046         dev = get_device(vid);
1047         if (unlikely(!dev))
1048                 return -1;
1049
1050         if (unlikely(!(dev->protocol_features &
1051             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1052                 return 0;
1053
1054         if (unlikely(!vq_is_packed(dev)))
1055                 return -1;
1056
1057         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1058                 return -1;
1059
1060         vq = dev->virtqueue[vring_idx];
1061         if (unlikely(!vq))
1062                 return -1;
1063
1064         inflight_info = vq->inflight_packed;
1065         if (unlikely(!inflight_info))
1066                 return -1;
1067
1068         if (unlikely(head >= vq->size))
1069                 return -1;
1070
1071         desc = vq->desc_packed;
1072         old_free_head = inflight_info->old_free_head;
1073         if (unlikely(old_free_head >= vq->size))
1074                 return -1;
1075
1076         free_head = old_free_head;
1077
1078         /* init header descriptor */
1079         inflight_info->desc[old_free_head].num = 0;
1080         inflight_info->desc[old_free_head].counter = vq->global_counter++;
1081         inflight_info->desc[old_free_head].inflight = 1;
1082
1083         /* save desc entry in flight entry */
1084         while (head != ((last + 1) % vq->size)) {
1085                 inflight_info->desc[old_free_head].num++;
1086                 inflight_info->desc[free_head].addr = desc[head].addr;
1087                 inflight_info->desc[free_head].len = desc[head].len;
1088                 inflight_info->desc[free_head].flags = desc[head].flags;
1089                 inflight_info->desc[free_head].id = desc[head].id;
1090
1091                 inflight_info->desc[old_free_head].last = free_head;
1092                 free_head = inflight_info->desc[free_head].next;
1093                 inflight_info->free_head = free_head;
1094                 head = (head + 1) % vq->size;
1095         }
1096
1097         inflight_info->old_free_head = free_head;
1098         *inflight_entry = old_free_head;
1099
1100         return 0;
1101 }
1102
1103 int
1104 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
1105                                   uint16_t last_used_idx, uint16_t idx)
1106 {
1107         struct virtio_net *dev;
1108         struct vhost_virtqueue *vq;
1109
1110         dev = get_device(vid);
1111         if (unlikely(!dev))
1112                 return -1;
1113
1114         if (unlikely(!(dev->protocol_features &
1115             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1116                 return 0;
1117
1118         if (unlikely(vq_is_packed(dev)))
1119                 return -1;
1120
1121         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1122                 return -1;
1123
1124         vq = dev->virtqueue[vring_idx];
1125         if (unlikely(!vq))
1126                 return -1;
1127
1128         if (unlikely(!vq->inflight_split))
1129                 return -1;
1130
1131         if (unlikely(idx >= vq->size))
1132                 return -1;
1133
1134         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1135
1136         vq->inflight_split->desc[idx].inflight = 0;
1137
1138         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1139
1140         vq->inflight_split->used_idx = last_used_idx;
1141         return 0;
1142 }
1143
1144 int
1145 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx,
1146                                    uint16_t head)
1147 {
1148         struct rte_vhost_inflight_info_packed *inflight_info;
1149         struct virtio_net *dev;
1150         struct vhost_virtqueue *vq;
1151
1152         dev = get_device(vid);
1153         if (unlikely(!dev))
1154                 return -1;
1155
1156         if (unlikely(!(dev->protocol_features &
1157             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1158                 return 0;
1159
1160         if (unlikely(!vq_is_packed(dev)))
1161                 return -1;
1162
1163         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1164                 return -1;
1165
1166         vq = dev->virtqueue[vring_idx];
1167         if (unlikely(!vq))
1168                 return -1;
1169
1170         inflight_info = vq->inflight_packed;
1171         if (unlikely(!inflight_info))
1172                 return -1;
1173
1174         if (unlikely(head >= vq->size))
1175                 return -1;
1176
1177         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1178
1179         inflight_info->desc[head].inflight = 0;
1180
1181         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1182
1183         inflight_info->old_free_head = inflight_info->free_head;
1184         inflight_info->old_used_idx = inflight_info->used_idx;
1185         inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter;
1186
1187         return 0;
1188 }
1189
1190 int
1191 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx,
1192                                      uint16_t idx)
1193 {
1194         struct virtio_net *dev;
1195         struct vhost_virtqueue *vq;
1196
1197         dev = get_device(vid);
1198         if (unlikely(!dev))
1199                 return -1;
1200
1201         if (unlikely(!(dev->protocol_features &
1202             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1203                 return 0;
1204
1205         if (unlikely(vq_is_packed(dev)))
1206                 return -1;
1207
1208         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1209                 return -1;
1210
1211         vq = dev->virtqueue[vring_idx];
1212         if (unlikely(!vq))
1213                 return -1;
1214
1215         if (unlikely(!vq->inflight_split))
1216                 return -1;
1217
1218         vq->inflight_split->last_inflight_io = idx;
1219         return 0;
1220 }
1221
1222 int
1223 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx,
1224                                       uint16_t head)
1225 {
1226         struct rte_vhost_inflight_info_packed *inflight_info;
1227         struct virtio_net *dev;
1228         struct vhost_virtqueue *vq;
1229         uint16_t last;
1230
1231         dev = get_device(vid);
1232         if (unlikely(!dev))
1233                 return -1;
1234
1235         if (unlikely(!(dev->protocol_features &
1236             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1237                 return 0;
1238
1239         if (unlikely(!vq_is_packed(dev)))
1240                 return -1;
1241
1242         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1243                 return -1;
1244
1245         vq = dev->virtqueue[vring_idx];
1246         if (unlikely(!vq))
1247                 return -1;
1248
1249         inflight_info = vq->inflight_packed;
1250         if (unlikely(!inflight_info))
1251                 return -1;
1252
1253         if (unlikely(head >= vq->size))
1254                 return -1;
1255
1256         last = inflight_info->desc[head].last;
1257         if (unlikely(last >= vq->size))
1258                 return -1;
1259
1260         inflight_info->desc[last].next = inflight_info->free_head;
1261         inflight_info->free_head = head;
1262         inflight_info->used_idx += inflight_info->desc[head].num;
1263         if (inflight_info->used_idx >= inflight_info->desc_num) {
1264                 inflight_info->used_idx -= inflight_info->desc_num;
1265                 inflight_info->used_wrap_counter =
1266                         !inflight_info->used_wrap_counter;
1267         }
1268
1269         return 0;
1270 }
1271
1272 int
1273 rte_vhost_vring_call(int vid, uint16_t vring_idx)
1274 {
1275         struct virtio_net *dev;
1276         struct vhost_virtqueue *vq;
1277
1278         dev = get_device(vid);
1279         if (!dev)
1280                 return -1;
1281
1282         if (vring_idx >= VHOST_MAX_VRING)
1283                 return -1;
1284
1285         vq = dev->virtqueue[vring_idx];
1286         if (!vq)
1287                 return -1;
1288
1289         if (vq_is_packed(dev))
1290                 vhost_vring_call_packed(dev, vq);
1291         else
1292                 vhost_vring_call_split(dev, vq);
1293
1294         return 0;
1295 }
1296
1297 uint16_t
1298 rte_vhost_avail_entries(int vid, uint16_t queue_id)
1299 {
1300         struct virtio_net *dev;
1301         struct vhost_virtqueue *vq;
1302         uint16_t ret = 0;
1303
1304         dev = get_device(vid);
1305         if (!dev)
1306                 return 0;
1307
1308         if (queue_id >= VHOST_MAX_VRING)
1309                 return 0;
1310
1311         vq = dev->virtqueue[queue_id];
1312         if (!vq)
1313                 return 0;
1314
1315         rte_spinlock_lock(&vq->access_lock);
1316
1317         if (unlikely(!vq->enabled || vq->avail == NULL))
1318                 goto out;
1319
1320         ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx;
1321
1322 out:
1323         rte_spinlock_unlock(&vq->access_lock);
1324         return ret;
1325 }
1326
1327 static inline int
1328 vhost_enable_notify_split(struct virtio_net *dev,
1329                 struct vhost_virtqueue *vq, int enable)
1330 {
1331         if (vq->used == NULL)
1332                 return -1;
1333
1334         if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
1335                 if (enable)
1336                         vq->used->flags &= ~VRING_USED_F_NO_NOTIFY;
1337                 else
1338                         vq->used->flags |= VRING_USED_F_NO_NOTIFY;
1339         } else {
1340                 if (enable)
1341                         vhost_avail_event(vq) = vq->last_avail_idx;
1342         }
1343         return 0;
1344 }
1345
1346 static inline int
1347 vhost_enable_notify_packed(struct virtio_net *dev,
1348                 struct vhost_virtqueue *vq, int enable)
1349 {
1350         uint16_t flags;
1351
1352         if (vq->device_event == NULL)
1353                 return -1;
1354
1355         if (!enable) {
1356                 vq->device_event->flags = VRING_EVENT_F_DISABLE;
1357                 return 0;
1358         }
1359
1360         flags = VRING_EVENT_F_ENABLE;
1361         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
1362                 flags = VRING_EVENT_F_DESC;
1363                 vq->device_event->off_wrap = vq->last_avail_idx |
1364                         vq->avail_wrap_counter << 15;
1365         }
1366
1367         rte_atomic_thread_fence(__ATOMIC_RELEASE);
1368
1369         vq->device_event->flags = flags;
1370         return 0;
1371 }
1372
1373 int
1374 vhost_enable_guest_notification(struct virtio_net *dev,
1375                 struct vhost_virtqueue *vq, int enable)
1376 {
1377         /*
1378          * If the virtqueue is not ready yet, it will be applied
1379          * when it will become ready.
1380          */
1381         if (!vq->ready)
1382                 return 0;
1383
1384         if (vq_is_packed(dev))
1385                 return vhost_enable_notify_packed(dev, vq, enable);
1386         else
1387                 return vhost_enable_notify_split(dev, vq, enable);
1388 }
1389
1390 int
1391 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
1392 {
1393         struct virtio_net *dev = get_device(vid);
1394         struct vhost_virtqueue *vq;
1395         int ret;
1396
1397         if (!dev)
1398                 return -1;
1399
1400         if (queue_id >= VHOST_MAX_VRING)
1401                 return -1;
1402
1403         vq = dev->virtqueue[queue_id];
1404         if (!vq)
1405                 return -1;
1406
1407         rte_spinlock_lock(&vq->access_lock);
1408
1409         vq->notif_enable = enable;
1410         ret = vhost_enable_guest_notification(dev, vq, enable);
1411
1412         rte_spinlock_unlock(&vq->access_lock);
1413
1414         return ret;
1415 }
1416
1417 void
1418 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len)
1419 {
1420         struct virtio_net *dev = get_device(vid);
1421
1422         if (dev == NULL)
1423                 return;
1424
1425         vhost_log_write(dev, addr, len);
1426 }
1427
1428 void
1429 rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
1430                          uint64_t offset, uint64_t len)
1431 {
1432         struct virtio_net *dev;
1433         struct vhost_virtqueue *vq;
1434
1435         dev = get_device(vid);
1436         if (dev == NULL)
1437                 return;
1438
1439         if (vring_idx >= VHOST_MAX_VRING)
1440                 return;
1441         vq = dev->virtqueue[vring_idx];
1442         if (!vq)
1443                 return;
1444
1445         vhost_log_used_vring(dev, vq, offset, len);
1446 }
1447
1448 uint32_t
1449 rte_vhost_rx_queue_count(int vid, uint16_t qid)
1450 {
1451         struct virtio_net *dev;
1452         struct vhost_virtqueue *vq;
1453         uint32_t ret = 0;
1454
1455         dev = get_device(vid);
1456         if (dev == NULL)
1457                 return 0;
1458
1459         if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) {
1460                 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
1461                         dev->vid, __func__, qid);
1462                 return 0;
1463         }
1464
1465         vq = dev->virtqueue[qid];
1466         if (vq == NULL)
1467                 return 0;
1468
1469         rte_spinlock_lock(&vq->access_lock);
1470
1471         if (unlikely(!vq->enabled || vq->avail == NULL))
1472                 goto out;
1473
1474         ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx;
1475
1476 out:
1477         rte_spinlock_unlock(&vq->access_lock);
1478         return ret;
1479 }
1480
1481 struct rte_vdpa_device *
1482 rte_vhost_get_vdpa_device(int vid)
1483 {
1484         struct virtio_net *dev = get_device(vid);
1485
1486         if (dev == NULL)
1487                 return NULL;
1488
1489         return dev->vdpa_dev;
1490 }
1491
1492 int rte_vhost_get_log_base(int vid, uint64_t *log_base,
1493                 uint64_t *log_size)
1494 {
1495         struct virtio_net *dev = get_device(vid);
1496
1497         if (dev == NULL || log_base == NULL || log_size == NULL)
1498                 return -1;
1499
1500         *log_base = dev->log_base;
1501         *log_size = dev->log_size;
1502
1503         return 0;
1504 }
1505
1506 int rte_vhost_get_vring_base(int vid, uint16_t queue_id,
1507                 uint16_t *last_avail_idx, uint16_t *last_used_idx)
1508 {
1509         struct vhost_virtqueue *vq;
1510         struct virtio_net *dev = get_device(vid);
1511
1512         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1513                 return -1;
1514
1515         if (queue_id >= VHOST_MAX_VRING)
1516                 return -1;
1517
1518         vq = dev->virtqueue[queue_id];
1519         if (!vq)
1520                 return -1;
1521
1522         if (vq_is_packed(dev)) {
1523                 *last_avail_idx = (vq->avail_wrap_counter << 15) |
1524                                   vq->last_avail_idx;
1525                 *last_used_idx = (vq->used_wrap_counter << 15) |
1526                                  vq->last_used_idx;
1527         } else {
1528                 *last_avail_idx = vq->last_avail_idx;
1529                 *last_used_idx = vq->last_used_idx;
1530         }
1531
1532         return 0;
1533 }
1534
1535 int rte_vhost_set_vring_base(int vid, uint16_t queue_id,
1536                 uint16_t last_avail_idx, uint16_t last_used_idx)
1537 {
1538         struct vhost_virtqueue *vq;
1539         struct virtio_net *dev = get_device(vid);
1540
1541         if (!dev)
1542                 return -1;
1543
1544         if (queue_id >= VHOST_MAX_VRING)
1545                 return -1;
1546
1547         vq = dev->virtqueue[queue_id];
1548         if (!vq)
1549                 return -1;
1550
1551         if (vq_is_packed(dev)) {
1552                 vq->last_avail_idx = last_avail_idx & 0x7fff;
1553                 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15));
1554                 vq->last_used_idx = last_used_idx & 0x7fff;
1555                 vq->used_wrap_counter = !!(last_used_idx & (1 << 15));
1556         } else {
1557                 vq->last_avail_idx = last_avail_idx;
1558                 vq->last_used_idx = last_used_idx;
1559         }
1560
1561         return 0;
1562 }
1563
1564 int
1565 rte_vhost_get_vring_base_from_inflight(int vid,
1566                                        uint16_t queue_id,
1567                                        uint16_t *last_avail_idx,
1568                                        uint16_t *last_used_idx)
1569 {
1570         struct rte_vhost_inflight_info_packed *inflight_info;
1571         struct vhost_virtqueue *vq;
1572         struct virtio_net *dev = get_device(vid);
1573
1574         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1575                 return -1;
1576
1577         if (queue_id >= VHOST_MAX_VRING)
1578                 return -1;
1579
1580         vq = dev->virtqueue[queue_id];
1581         if (!vq)
1582                 return -1;
1583
1584         if (!vq_is_packed(dev))
1585                 return -1;
1586
1587         inflight_info = vq->inflight_packed;
1588         if (!inflight_info)
1589                 return -1;
1590
1591         *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) |
1592                           inflight_info->old_used_idx;
1593         *last_used_idx = *last_avail_idx;
1594
1595         return 0;
1596 }
1597
1598 int rte_vhost_extern_callback_register(int vid,
1599                 struct rte_vhost_user_extern_ops const * const ops, void *ctx)
1600 {
1601         struct virtio_net *dev = get_device(vid);
1602
1603         if (dev == NULL || ops == NULL)
1604                 return -1;
1605
1606         dev->extern_ops = *ops;
1607         dev->extern_data = ctx;
1608         return 0;
1609 }
1610
1611 int rte_vhost_async_channel_register(int vid, uint16_t queue_id,
1612                                         uint32_t features,
1613                                         struct rte_vhost_async_channel_ops *ops)
1614 {
1615         struct vhost_virtqueue *vq;
1616         struct virtio_net *dev = get_device(vid);
1617         struct rte_vhost_async_features f;
1618         int node;
1619
1620         if (dev == NULL || ops == NULL)
1621                 return -1;
1622
1623         f.intval = features;
1624
1625         if (queue_id >= VHOST_MAX_VRING)
1626                 return -1;
1627
1628         vq = dev->virtqueue[queue_id];
1629
1630         if (unlikely(vq == NULL || !dev->async_copy))
1631                 return -1;
1632
1633         if (unlikely(!f.async_inorder)) {
1634                 VHOST_LOG_CONFIG(ERR,
1635                         "async copy is not supported on non-inorder mode "
1636                         "(vid %d, qid: %d)\n", vid, queue_id);
1637                 return -1;
1638         }
1639
1640         if (unlikely(ops->check_completed_copies == NULL ||
1641                 ops->transfer_data == NULL))
1642                 return -1;
1643
1644         rte_spinlock_lock(&vq->access_lock);
1645
1646         if (unlikely(vq->async_registered)) {
1647                 VHOST_LOG_CONFIG(ERR,
1648                         "async register failed: channel already registered "
1649                         "(vid %d, qid: %d)\n", vid, queue_id);
1650                 goto reg_out;
1651         }
1652
1653 #ifdef RTE_LIBRTE_VHOST_NUMA
1654         if (get_mempolicy(&node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) {
1655                 VHOST_LOG_CONFIG(ERR,
1656                         "unable to get numa information in async register. "
1657                         "allocating async buffer memory on the caller thread node\n");
1658                 node = SOCKET_ID_ANY;
1659         }
1660 #else
1661         node = SOCKET_ID_ANY;
1662 #endif
1663
1664         vq->async_pkts_info = rte_malloc_socket(NULL,
1665                         vq->size * sizeof(struct async_inflight_info),
1666                         RTE_CACHE_LINE_SIZE, node);
1667         if (!vq->async_pkts_info) {
1668                 vhost_free_async_mem(vq);
1669                 VHOST_LOG_CONFIG(ERR,
1670                         "async register failed: cannot allocate memory for async_pkts_info "
1671                         "(vid %d, qid: %d)\n", vid, queue_id);
1672                 goto reg_out;
1673         }
1674
1675         vq->it_pool = rte_malloc_socket(NULL,
1676                         VHOST_MAX_ASYNC_IT * sizeof(struct rte_vhost_iov_iter),
1677                         RTE_CACHE_LINE_SIZE, node);
1678         if (!vq->it_pool) {
1679                 vhost_free_async_mem(vq);
1680                 VHOST_LOG_CONFIG(ERR,
1681                         "async register failed: cannot allocate memory for it_pool "
1682                         "(vid %d, qid: %d)\n", vid, queue_id);
1683                 goto reg_out;
1684         }
1685
1686         vq->vec_pool = rte_malloc_socket(NULL,
1687                         VHOST_MAX_ASYNC_VEC * sizeof(struct iovec),
1688                         RTE_CACHE_LINE_SIZE, node);
1689         if (!vq->vec_pool) {
1690                 vhost_free_async_mem(vq);
1691                 VHOST_LOG_CONFIG(ERR,
1692                         "async register failed: cannot allocate memory for vec_pool "
1693                         "(vid %d, qid: %d)\n", vid, queue_id);
1694                 goto reg_out;
1695         }
1696
1697         if (vq_is_packed(dev)) {
1698                 vq->async_buffers_packed = rte_malloc_socket(NULL,
1699                         vq->size * sizeof(struct vring_used_elem_packed),
1700                         RTE_CACHE_LINE_SIZE, node);
1701                 if (!vq->async_buffers_packed) {
1702                         vhost_free_async_mem(vq);
1703                         VHOST_LOG_CONFIG(ERR,
1704                                 "async register failed: cannot allocate memory for async buffers "
1705                                 "(vid %d, qid: %d)\n", vid, queue_id);
1706                         goto reg_out;
1707                 }
1708         } else {
1709                 vq->async_descs_split = rte_malloc_socket(NULL,
1710                         vq->size * sizeof(struct vring_used_elem),
1711                         RTE_CACHE_LINE_SIZE, node);
1712                 if (!vq->async_descs_split) {
1713                         vhost_free_async_mem(vq);
1714                         VHOST_LOG_CONFIG(ERR,
1715                                 "async register failed: cannot allocate memory for async descs "
1716                                 "(vid %d, qid: %d)\n", vid, queue_id);
1717                         goto reg_out;
1718                 }
1719         }
1720
1721         vq->async_ops.check_completed_copies = ops->check_completed_copies;
1722         vq->async_ops.transfer_data = ops->transfer_data;
1723
1724         vq->async_inorder = f.async_inorder;
1725         vq->async_threshold = f.async_threshold;
1726
1727         vq->async_registered = true;
1728
1729 reg_out:
1730         rte_spinlock_unlock(&vq->access_lock);
1731
1732         return 0;
1733 }
1734
1735 int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id)
1736 {
1737         struct vhost_virtqueue *vq;
1738         struct virtio_net *dev = get_device(vid);
1739         int ret = -1;
1740
1741         if (dev == NULL)
1742                 return ret;
1743
1744         if (queue_id >= VHOST_MAX_VRING)
1745                 return ret;
1746
1747         vq = dev->virtqueue[queue_id];
1748
1749         if (vq == NULL)
1750                 return ret;
1751
1752         ret = 0;
1753
1754         if (!vq->async_registered)
1755                 return ret;
1756
1757         if (!rte_spinlock_trylock(&vq->access_lock)) {
1758                 VHOST_LOG_CONFIG(ERR, "Failed to unregister async channel. "
1759                         "virt queue busy.\n");
1760                 return -1;
1761         }
1762
1763         if (vq->async_pkts_inflight_n) {
1764                 VHOST_LOG_CONFIG(ERR, "Failed to unregister async channel. "
1765                         "async inflight packets must be completed before unregistration.\n");
1766                 ret = -1;
1767                 goto out;
1768         }
1769
1770         vhost_free_async_mem(vq);
1771
1772         vq->async_ops.transfer_data = NULL;
1773         vq->async_ops.check_completed_copies = NULL;
1774         vq->async_registered = false;
1775
1776 out:
1777         rte_spinlock_unlock(&vq->access_lock);
1778
1779         return ret;
1780 }
1781
1782 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO);
1783 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING);