lib/vhost/vhost.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <linux/vhost.h>
   6 #include <linux/virtio_net.h>
   7 #include <stddef.h>
   8 #include <stdint.h>
   9 #include <stdlib.h>
  10 #ifdef RTE_LIBRTE_VHOST_NUMA
  11 #include <numa.h>
  12 #include <numaif.h>
  13 #endif
  14
  15 #include <rte_errno.h>
  16 #include <rte_ethdev.h>
  17 #include <rte_log.h>
  18 #include <rte_string_fns.h>
  19 #include <rte_memory.h>
  20 #include <rte_malloc.h>
  21 #include <rte_vhost.h>
  22 #include <rte_rwlock.h>
  23
  24 #include "iotlb.h"
  25 #include "vhost.h"
  26 #include "vhost_user.h"
  27
  28 struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
  29 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER;
  30
  31 /* Called with iotlb_lock read-locked */
  32 uint64_t
  33 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
  34                     uint64_t iova, uint64_t *size, uint8_t perm)
  35 {
  36         uint64_t vva, tmp_size;
  37
  38         if (unlikely(!*size))
  39                 return 0;
  40
  41         tmp_size = *size;
  42
  43         vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm);
  44         if (tmp_size == *size)
  45                 return vva;
  46
  47         iova += tmp_size;
  48
  49         if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) {
  50                 /*
  51                  * iotlb_lock is read-locked for a full burst,
  52                  * but it only protects the iotlb cache.
  53                  * In case of IOTLB miss, we might block on the socket,
  54                  * which could cause a deadlock with QEMU if an IOTLB update
  55                  * is being handled. We can safely unlock here to avoid it.
  56                  */
  57                 vhost_user_iotlb_rd_unlock(vq);
  58
  59                 vhost_user_iotlb_pending_insert(dev, vq, iova, perm);
  60                 if (vhost_user_iotlb_miss(dev, iova, perm)) {
  61                         VHOST_LOG_DATA(ERR, "(%s) IOTLB miss req failed for IOVA 0x%" PRIx64 "\n",
  62                                 dev->ifname, iova);
  63                         vhost_user_iotlb_pending_remove(vq, iova, 1, perm);
  64                 }
  65
  66                 vhost_user_iotlb_rd_lock(vq);
  67         }
  68
  69         return 0;
  70 }
  71
  72 #define VHOST_LOG_PAGE  4096
  73
  74 /*
  75  * Atomically set a bit in memory.
  76  */
  77 static __rte_always_inline void
  78 vhost_set_bit(unsigned int nr, volatile uint8_t *addr)
  79 {
  80 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
  81         /*
  82          * __sync_ built-ins are deprecated, but __atomic_ ones
  83          * are sub-optimized in older GCC versions.
  84          */
  85         __sync_fetch_and_or_1(addr, (1U << nr));
  86 #else
  87         __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED);
  88 #endif
  89 }
  90
  91 static __rte_always_inline void
  92 vhost_log_page(uint8_t *log_base, uint64_t page)
  93 {
  94         vhost_set_bit(page % 8, &log_base[page / 8]);
  95 }
  96
  97 void
  98 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
  99 {
 100         uint64_t page;
 101
 102         if (unlikely(!dev->log_base || !len))
 103                 return;
 104
 105         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 106                 return;
 107
 108         /* To make sure guest memory updates are committed before logging */
 109         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 110
 111         page = addr / VHOST_LOG_PAGE;
 112         while (page * VHOST_LOG_PAGE < addr + len) {
 113                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 114                 page += 1;
 115         }
 116 }
 117
 118 void
 119 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 120                              uint64_t iova, uint64_t len)
 121 {
 122         uint64_t hva, gpa, map_len;
 123         map_len = len;
 124
 125         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 126         if (map_len != len) {
 127                 VHOST_LOG_DATA(ERR,
 128                         "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 129                         dev->ifname, iova);
 130                 return;
 131         }
 132
 133         gpa = hva_to_gpa(dev, hva, len);
 134         if (gpa)
 135                 __vhost_log_write(dev, gpa, len);
 136 }
 137
 138 void
 139 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
 140 {
 141         unsigned long *log_base;
 142         int i;
 143
 144         if (unlikely(!dev->log_base))
 145                 return;
 146
 147         /* No cache, nothing to sync */
 148         if (unlikely(!vq->log_cache))
 149                 return;
 150
 151         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 152
 153         log_base = (unsigned long *)(uintptr_t)dev->log_base;
 154
 155         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 156                 struct log_cache_entry *elem = vq->log_cache + i;
 157
 158 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
 159                 /*
 160                  * '__sync' builtins are deprecated, but '__atomic' ones
 161                  * are sub-optimized in older GCC versions.
 162                  */
 163                 __sync_fetch_and_or(log_base + elem->offset, elem->val);
 164 #else
 165                 __atomic_fetch_or(log_base + elem->offset, elem->val,
 166                                 __ATOMIC_RELAXED);
 167 #endif
 168         }
 169
 170         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 171
 172         vq->log_cache_nb_elem = 0;
 173 }
 174
 175 static __rte_always_inline void
 176 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
 177                         uint64_t page)
 178 {
 179         uint32_t bit_nr = page % (sizeof(unsigned long) << 3);
 180         uint32_t offset = page / (sizeof(unsigned long) << 3);
 181         int i;
 182
 183         if (unlikely(!vq->log_cache)) {
 184                 /* No logging cache allocated, write dirty log map directly */
 185                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 186                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 187
 188                 return;
 189         }
 190
 191         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 192                 struct log_cache_entry *elem = vq->log_cache + i;
 193
 194                 if (elem->offset == offset) {
 195                         elem->val |= (1UL << bit_nr);
 196                         return;
 197                 }
 198         }
 199
 200         if (unlikely(i >= VHOST_LOG_CACHE_NR)) {
 201                 /*
 202                  * No more room for a new log cache entry,
 203                  * so write the dirty log map directly.
 204                  */
 205                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 206                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 207
 208                 return;
 209         }
 210
 211         vq->log_cache[i].offset = offset;
 212         vq->log_cache[i].val = (1UL << bit_nr);
 213         vq->log_cache_nb_elem++;
 214 }
 215
 216 void
 217 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
 218                         uint64_t addr, uint64_t len)
 219 {
 220         uint64_t page;
 221
 222         if (unlikely(!dev->log_base || !len))
 223                 return;
 224
 225         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 226                 return;
 227
 228         page = addr / VHOST_LOG_PAGE;
 229         while (page * VHOST_LOG_PAGE < addr + len) {
 230                 vhost_log_cache_page(dev, vq, page);
 231                 page += 1;
 232         }
 233 }
 234
 235 void
 236 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 237                              uint64_t iova, uint64_t len)
 238 {
 239         uint64_t hva, gpa, map_len;
 240         map_len = len;
 241
 242         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 243         if (map_len != len) {
 244                 VHOST_LOG_DATA(ERR,
 245                         "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 246                         dev->ifname, iova);
 247                 return;
 248         }
 249
 250         gpa = hva_to_gpa(dev, hva, len);
 251         if (gpa)
 252                 __vhost_log_cache_write(dev, vq, gpa, len);
 253 }
 254
 255 void *
 256 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
 257                 uint64_t desc_addr, uint64_t desc_len)
 258 {
 259         void *idesc;
 260         uint64_t src, dst;
 261         uint64_t len, remain = desc_len;
 262
 263         idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node);
 264         if (unlikely(!idesc))
 265                 return NULL;
 266
 267         dst = (uint64_t)(uintptr_t)idesc;
 268
 269         while (remain) {
 270                 len = remain;
 271                 src = vhost_iova_to_vva(dev, vq, desc_addr, &len,
 272                                 VHOST_ACCESS_RO);
 273                 if (unlikely(!src || !len)) {
 274                         rte_free(idesc);
 275                         return NULL;
 276                 }
 277
 278                 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len);
 279
 280                 remain -= len;
 281                 dst += len;
 282                 desc_addr += len;
 283         }
 284
 285         return idesc;
 286 }
 287
 288 void
 289 cleanup_vq(struct vhost_virtqueue *vq, int destroy)
 290 {
 291         if ((vq->callfd >= 0) && (destroy != 0))
 292                 close(vq->callfd);
 293         if (vq->kickfd >= 0)
 294                 close(vq->kickfd);
 295 }
 296
 297 void
 298 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq)
 299 {
 300         if (!(dev->protocol_features &
 301             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
 302                 return;
 303
 304         if (vq_is_packed(dev)) {
 305                 if (vq->inflight_packed)
 306                         vq->inflight_packed = NULL;
 307         } else {
 308                 if (vq->inflight_split)
 309                         vq->inflight_split = NULL;
 310         }
 311
 312         if (vq->resubmit_inflight) {
 313                 if (vq->resubmit_inflight->resubmit_list) {
 314                         rte_free(vq->resubmit_inflight->resubmit_list);
 315                         vq->resubmit_inflight->resubmit_list = NULL;
 316                 }
 317                 rte_free(vq->resubmit_inflight);
 318                 vq->resubmit_inflight = NULL;
 319         }
 320 }
 321
 322 /*
 323  * Unmap any memory, close any file descriptors and
 324  * free any memory owned by a device.
 325  */
 326 void
 327 cleanup_device(struct virtio_net *dev, int destroy)
 328 {
 329         uint32_t i;
 330
 331         vhost_backend_cleanup(dev);
 332
 333         for (i = 0; i < dev->nr_vring; i++) {
 334                 cleanup_vq(dev->virtqueue[i], destroy);
 335                 cleanup_vq_inflight(dev, dev->virtqueue[i]);
 336         }
 337 }
 338
 339 static void
 340 vhost_free_async_mem(struct vhost_virtqueue *vq)
 341 {
 342         if (!vq->async)
 343                 return;
 344
 345         rte_free(vq->async->pkts_info);
 346
 347         rte_free(vq->async->buffers_packed);
 348         vq->async->buffers_packed = NULL;
 349         rte_free(vq->async->descs_split);
 350         vq->async->descs_split = NULL;
 351
 352         rte_free(vq->async);
 353         vq->async = NULL;
 354 }
 355
 356 void
 357 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 358 {
 359         if (vq_is_packed(dev))
 360                 rte_free(vq->shadow_used_packed);
 361         else
 362                 rte_free(vq->shadow_used_split);
 363
 364         vhost_free_async_mem(vq);
 365         rte_free(vq->batch_copy_elems);
 366         rte_mempool_free(vq->iotlb_pool);
 367         rte_free(vq->log_cache);
 368         rte_free(vq);
 369 }
 370
 371 /*
 372  * Release virtqueues and device memory.
 373  */
 374 static void
 375 free_device(struct virtio_net *dev)
 376 {
 377         uint32_t i;
 378
 379         for (i = 0; i < dev->nr_vring; i++)
 380                 free_vq(dev, dev->virtqueue[i]);
 381
 382         rte_free(dev);
 383 }
 384
 385 static __rte_always_inline int
 386 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 387 {
 388         if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG))))
 389                 return 0;
 390
 391         vq->log_guest_addr = translate_log_addr(dev, vq,
 392                                                 vq->ring_addrs.log_guest_addr);
 393         if (vq->log_guest_addr == 0)
 394                 return -1;
 395
 396         return 0;
 397 }
 398
 399 /*
 400  * Converts vring log address to GPA
 401  * If IOMMU is enabled, the log address is IOVA
 402  * If IOMMU not enabled, the log address is already GPA
 403  *
 404  * Caller should have iotlb_lock read-locked
 405  */
 406 uint64_t
 407 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 408                 uint64_t log_addr)
 409 {
 410         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 411                 const uint64_t exp_size = sizeof(uint64_t);
 412                 uint64_t hva, gpa;
 413                 uint64_t size = exp_size;
 414
 415                 hva = vhost_iova_to_vva(dev, vq, log_addr,
 416                                         &size, VHOST_ACCESS_RW);
 417
 418                 if (size != exp_size)
 419                         return 0;
 420
 421                 gpa = hva_to_gpa(dev, hva, exp_size);
 422                 if (!gpa) {
 423                         VHOST_LOG_DATA(ERR,
 424                                 "(%s) failed to find GPA for log_addr: 0x%"
 425                                 PRIx64 " hva: 0x%" PRIx64 "\n",
 426                                 dev->ifname, log_addr, hva);
 427                         return 0;
 428                 }
 429                 return gpa;
 430
 431         } else
 432                 return log_addr;
 433 }
 434
 435 /* Caller should have iotlb_lock read-locked */
 436 static int
 437 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 438 {
 439         uint64_t req_size, size;
 440
 441         req_size = sizeof(struct vring_desc) * vq->size;
 442         size = req_size;
 443         vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 444                                                 vq->ring_addrs.desc_user_addr,
 445                                                 &size, VHOST_ACCESS_RW);
 446         if (!vq->desc || size != req_size)
 447                 return -1;
 448
 449         req_size = sizeof(struct vring_avail);
 450         req_size += sizeof(uint16_t) * vq->size;
 451         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 452                 req_size += sizeof(uint16_t);
 453         size = req_size;
 454         vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 455                                                 vq->ring_addrs.avail_user_addr,
 456                                                 &size, VHOST_ACCESS_RW);
 457         if (!vq->avail || size != req_size)
 458                 return -1;
 459
 460         req_size = sizeof(struct vring_used);
 461         req_size += sizeof(struct vring_used_elem) * vq->size;
 462         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 463                 req_size += sizeof(uint16_t);
 464         size = req_size;
 465         vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 466                                                 vq->ring_addrs.used_user_addr,
 467                                                 &size, VHOST_ACCESS_RW);
 468         if (!vq->used || size != req_size)
 469                 return -1;
 470
 471         return 0;
 472 }
 473
 474 /* Caller should have iotlb_lock read-locked */
 475 static int
 476 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)
 477 {
 478         uint64_t req_size, size;
 479
 480         req_size = sizeof(struct vring_packed_desc) * vq->size;
 481         size = req_size;
 482         vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 483                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr,
 484                                 &size, VHOST_ACCESS_RW);
 485         if (!vq->desc_packed || size != req_size)
 486                 return -1;
 487
 488         req_size = sizeof(struct vring_packed_desc_event);
 489         size = req_size;
 490         vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t)
 491                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr,
 492                                 &size, VHOST_ACCESS_RW);
 493         if (!vq->driver_event || size != req_size)
 494                 return -1;
 495
 496         req_size = sizeof(struct vring_packed_desc_event);
 497         size = req_size;
 498         vq->device_event = (struct vring_packed_desc_event *)(uintptr_t)
 499                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr,
 500                                 &size, VHOST_ACCESS_RW);
 501         if (!vq->device_event || size != req_size)
 502                 return -1;
 503
 504         return 0;
 505 }
 506
 507 int
 508 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 509 {
 510
 511         if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
 512                 return -1;
 513
 514         if (vq_is_packed(dev)) {
 515                 if (vring_translate_packed(dev, vq) < 0)
 516                         return -1;
 517         } else {
 518                 if (vring_translate_split(dev, vq) < 0)
 519                         return -1;
 520         }
 521
 522         if (log_translate(dev, vq) < 0)
 523                 return -1;
 524
 525         vq->access_ok = true;
 526
 527         return 0;
 528 }
 529
 530 void
 531 vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 532 {
 533         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 534                 vhost_user_iotlb_wr_lock(vq);
 535
 536         vq->access_ok = false;
 537         vq->desc = NULL;
 538         vq->avail = NULL;
 539         vq->used = NULL;
 540         vq->log_guest_addr = 0;
 541
 542         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 543                 vhost_user_iotlb_wr_unlock(vq);
 544 }
 545
 546 static void
 547 init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 548 {
 549         struct vhost_virtqueue *vq;
 550         int numa_node = SOCKET_ID_ANY;
 551
 552         if (vring_idx >= VHOST_MAX_VRING) {
 553                 VHOST_LOG_CONFIG(ERR, "(%s) failed to init vring, out of bound (%d)\n",
 554                                 dev->ifname, vring_idx);
 555                 return;
 556         }
 557
 558         vq = dev->virtqueue[vring_idx];
 559         if (!vq) {
 560                 VHOST_LOG_CONFIG(ERR, "(%s) virtqueue not allocated (%d)\n",
 561                                 dev->ifname, vring_idx);
 562                 return;
 563         }
 564
 565         memset(vq, 0, sizeof(struct vhost_virtqueue));
 566
 567         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
 568         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
 569         vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
 570
 571 #ifdef RTE_LIBRTE_VHOST_NUMA
 572         if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) {
 573                 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n",
 574                         dev->ifname, rte_strerror(errno));
 575                 numa_node = SOCKET_ID_ANY;
 576         }
 577 #endif
 578         vq->numa_node = numa_node;
 579
 580         vhost_user_iotlb_init(dev, vring_idx);
 581 }
 582
 583 static void
 584 reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 585 {
 586         struct vhost_virtqueue *vq;
 587         int callfd;
 588
 589         if (vring_idx >= VHOST_MAX_VRING) {
 590                 VHOST_LOG_CONFIG(ERR,
 591                                 "(%s) failed to reset vring, out of bound (%d)\n",
 592                                 dev->ifname, vring_idx);
 593                 return;
 594         }
 595
 596         vq = dev->virtqueue[vring_idx];
 597         if (!vq) {
 598                 VHOST_LOG_CONFIG(ERR, "(%s) failed to reset vring, virtqueue not allocated (%d)\n",
 599                                 dev->ifname, vring_idx);
 600                 return;
 601         }
 602
 603         callfd = vq->callfd;
 604         init_vring_queue(dev, vring_idx);
 605         vq->callfd = callfd;
 606 }
 607
 608 int
 609 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 610 {
 611         struct vhost_virtqueue *vq;
 612         uint32_t i;
 613
 614         /* Also allocate holes, if any, up to requested vring index. */
 615         for (i = 0; i <= vring_idx; i++) {
 616                 if (dev->virtqueue[i])
 617                         continue;
 618
 619                 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0);
 620                 if (vq == NULL) {
 621                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate memory for vring %u.\n",
 622                                         dev->ifname, i);
 623                         return -1;
 624                 }
 625
 626                 dev->virtqueue[i] = vq;
 627                 init_vring_queue(dev, i);
 628                 rte_spinlock_init(&vq->access_lock);
 629                 vq->avail_wrap_counter = 1;
 630                 vq->used_wrap_counter = 1;
 631                 vq->signalled_used_valid = false;
 632         }
 633
 634         dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1);
 635
 636         return 0;
 637 }
 638
 639 /*
 640  * Reset some variables in device structure, while keeping few
 641  * others untouched, such as vid, ifname, nr_vring: they
 642  * should be same unless the device is removed.
 643  */
 644 void
 645 reset_device(struct virtio_net *dev)
 646 {
 647         uint32_t i;
 648
 649         dev->features = 0;
 650         dev->protocol_features = 0;
 651         dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 652
 653         for (i = 0; i < dev->nr_vring; i++)
 654                 reset_vring_queue(dev, i);
 655 }
 656
 657 /*
 658  * Invoked when there is a new vhost-user connection established (when
 659  * there is a new virtio device being attached).
 660  */
 661 int
 662 vhost_new_device(void)
 663 {
 664         struct virtio_net *dev;
 665         int i;
 666
 667         pthread_mutex_lock(&vhost_dev_lock);
 668         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
 669                 if (vhost_devices[i] == NULL)
 670                         break;
 671         }
 672
 673         if (i == MAX_VHOST_DEVICE) {
 674                 VHOST_LOG_CONFIG(ERR, "failed to find a free slot for new device.\n");
 675                 pthread_mutex_unlock(&vhost_dev_lock);
 676                 return -1;
 677         }
 678
 679         dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0);
 680         if (dev == NULL) {
 681                 VHOST_LOG_CONFIG(ERR, "failed to allocate memory for new device.\n");
 682                 pthread_mutex_unlock(&vhost_dev_lock);
 683                 return -1;
 684         }
 685
 686         vhost_devices[i] = dev;
 687         pthread_mutex_unlock(&vhost_dev_lock);
 688
 689         dev->vid = i;
 690         dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 691         dev->slave_req_fd = -1;
 692         dev->postcopy_ufd = -1;
 693         rte_spinlock_init(&dev->slave_req_lock);
 694
 695         return i;
 696 }
 697
 698 void
 699 vhost_destroy_device_notify(struct virtio_net *dev)
 700 {
 701         struct rte_vdpa_device *vdpa_dev;
 702
 703         if (dev->flags & VIRTIO_DEV_RUNNING) {
 704                 vdpa_dev = dev->vdpa_dev;
 705                 if (vdpa_dev)
 706                         vdpa_dev->ops->dev_close(dev->vid);
 707                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 708                 dev->notify_ops->destroy_device(dev->vid);
 709         }
 710 }
 711
 712 /*
 713  * Invoked when there is the vhost-user connection is broken (when
 714  * the virtio device is being detached).
 715  */
 716 void
 717 vhost_destroy_device(int vid)
 718 {
 719         struct virtio_net *dev = get_device(vid);
 720
 721         if (dev == NULL)
 722                 return;
 723
 724         vhost_destroy_device_notify(dev);
 725
 726         cleanup_device(dev, 1);
 727         free_device(dev);
 728
 729         vhost_devices[vid] = NULL;
 730 }
 731
 732 void
 733 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev)
 734 {
 735         struct virtio_net *dev = get_device(vid);
 736
 737         if (dev == NULL)
 738                 return;
 739
 740         dev->vdpa_dev = vdpa_dev;
 741 }
 742
 743 void
 744 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len)
 745 {
 746         struct virtio_net *dev;
 747         unsigned int len;
 748
 749         dev = get_device(vid);
 750         if (dev == NULL)
 751                 return;
 752
 753         len = if_len > sizeof(dev->ifname) ?
 754                 sizeof(dev->ifname) : if_len;
 755
 756         strncpy(dev->ifname, if_name, len);
 757         dev->ifname[sizeof(dev->ifname) - 1] = '\0';
 758 }
 759
 760 void
 761 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags)
 762 {
 763         struct virtio_net *dev = get_device(vid);
 764
 765         if (dev == NULL)
 766                 return;
 767
 768         if (enable)
 769                 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 770         else
 771                 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 772         if (!compliant_ol_flags)
 773                 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS;
 774         else
 775                 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS;
 776 }
 777
 778 void
 779 vhost_enable_extbuf(int vid)
 780 {
 781         struct virtio_net *dev = get_device(vid);
 782
 783         if (dev == NULL)
 784                 return;
 785
 786         dev->extbuf = 1;
 787 }
 788
 789 void
 790 vhost_enable_linearbuf(int vid)
 791 {
 792         struct virtio_net *dev = get_device(vid);
 793
 794         if (dev == NULL)
 795                 return;
 796
 797         dev->linearbuf = 1;
 798 }
 799
 800 int
 801 rte_vhost_get_mtu(int vid, uint16_t *mtu)
 802 {
 803         struct virtio_net *dev = get_device(vid);
 804
 805         if (dev == NULL || mtu == NULL)
 806                 return -ENODEV;
 807
 808         if (!(dev->flags & VIRTIO_DEV_READY))
 809                 return -EAGAIN;
 810
 811         if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU)))
 812                 return -ENOTSUP;
 813
 814         *mtu = dev->mtu;
 815
 816         return 0;
 817 }
 818
 819 int
 820 rte_vhost_get_numa_node(int vid)
 821 {
 822 #ifdef RTE_LIBRTE_VHOST_NUMA
 823         struct virtio_net *dev = get_device(vid);
 824         int numa_node;
 825         int ret;
 826
 827         if (dev == NULL || numa_available() != 0)
 828                 return -1;
 829
 830         ret = get_mempolicy(&numa_node, NULL, 0, dev,
 831                             MPOL_F_NODE | MPOL_F_ADDR);
 832         if (ret < 0) {
 833                 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n",
 834                         dev->ifname, rte_strerror(errno));
 835                 return -1;
 836         }
 837
 838         return numa_node;
 839 #else
 840         RTE_SET_USED(vid);
 841         return -1;
 842 #endif
 843 }
 844
 845 uint32_t
 846 rte_vhost_get_queue_num(int vid)
 847 {
 848         struct virtio_net *dev = get_device(vid);
 849
 850         if (dev == NULL)
 851                 return 0;
 852
 853         return dev->nr_vring / 2;
 854 }
 855
 856 uint16_t
 857 rte_vhost_get_vring_num(int vid)
 858 {
 859         struct virtio_net *dev = get_device(vid);
 860
 861         if (dev == NULL)
 862                 return 0;
 863
 864         return dev->nr_vring;
 865 }
 866
 867 int
 868 rte_vhost_get_ifname(int vid, char *buf, size_t len)
 869 {
 870         struct virtio_net *dev = get_device(vid);
 871
 872         if (dev == NULL || buf == NULL)
 873                 return -1;
 874
 875         len = RTE_MIN(len, sizeof(dev->ifname));
 876
 877         strncpy(buf, dev->ifname, len);
 878         buf[len - 1] = '\0';
 879
 880         return 0;
 881 }
 882
 883 int
 884 rte_vhost_get_negotiated_features(int vid, uint64_t *features)
 885 {
 886         struct virtio_net *dev;
 887
 888         dev = get_device(vid);
 889         if (dev == NULL || features == NULL)
 890                 return -1;
 891
 892         *features = dev->features;
 893         return 0;
 894 }
 895
 896 int
 897 rte_vhost_get_negotiated_protocol_features(int vid,
 898                                            uint64_t *protocol_features)
 899 {
 900         struct virtio_net *dev;
 901
 902         dev = get_device(vid);
 903         if (dev == NULL || protocol_features == NULL)
 904                 return -1;
 905
 906         *protocol_features = dev->protocol_features;
 907         return 0;
 908 }
 909
 910 int
 911 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem)
 912 {
 913         struct virtio_net *dev;
 914         struct rte_vhost_memory *m;
 915         size_t size;
 916
 917         dev = get_device(vid);
 918         if (dev == NULL || mem == NULL)
 919                 return -1;
 920
 921         size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region);
 922         m = malloc(sizeof(struct rte_vhost_memory) + size);
 923         if (!m)
 924                 return -1;
 925
 926         m->nregions = dev->mem->nregions;
 927         memcpy(m->regions, dev->mem->regions, size);
 928         *mem = m;
 929
 930         return 0;
 931 }
 932
 933 int
 934 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
 935                           struct rte_vhost_vring *vring)
 936 {
 937         struct virtio_net *dev;
 938         struct vhost_virtqueue *vq;
 939
 940         dev = get_device(vid);
 941         if (dev == NULL || vring == NULL)
 942                 return -1;
 943
 944         if (vring_idx >= VHOST_MAX_VRING)
 945                 return -1;
 946
 947         vq = dev->virtqueue[vring_idx];
 948         if (!vq)
 949                 return -1;
 950
 951         if (vq_is_packed(dev)) {
 952                 vring->desc_packed = vq->desc_packed;
 953                 vring->driver_event = vq->driver_event;
 954                 vring->device_event = vq->device_event;
 955         } else {
 956                 vring->desc = vq->desc;
 957                 vring->avail = vq->avail;
 958                 vring->used = vq->used;
 959         }
 960         vring->log_guest_addr  = vq->log_guest_addr;
 961
 962         vring->callfd  = vq->callfd;
 963         vring->kickfd  = vq->kickfd;
 964         vring->size    = vq->size;
 965
 966         return 0;
 967 }
 968
 969 int
 970 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx,
 971                                   struct rte_vhost_ring_inflight *vring)
 972 {
 973         struct virtio_net *dev;
 974         struct vhost_virtqueue *vq;
 975
 976         dev = get_device(vid);
 977         if (unlikely(!dev))
 978                 return -1;
 979
 980         if (vring_idx >= VHOST_MAX_VRING)
 981                 return -1;
 982
 983         vq = dev->virtqueue[vring_idx];
 984         if (unlikely(!vq))
 985                 return -1;
 986
 987         if (vq_is_packed(dev)) {
 988                 if (unlikely(!vq->inflight_packed))
 989                         return -1;
 990
 991                 vring->inflight_packed = vq->inflight_packed;
 992         } else {
 993                 if (unlikely(!vq->inflight_split))
 994                         return -1;
 995
 996                 vring->inflight_split = vq->inflight_split;
 997         }
 998
 999         vring->resubmit_inflight = vq->resubmit_inflight;
1000
1001         return 0;
1002 }
1003
1004 int
1005 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx,
1006                                   uint16_t idx)
1007 {
1008         struct vhost_virtqueue *vq;
1009         struct virtio_net *dev;
1010
1011         dev = get_device(vid);
1012         if (unlikely(!dev))
1013                 return -1;
1014
1015         if (unlikely(!(dev->protocol_features &
1016             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1017                 return 0;
1018
1019         if (unlikely(vq_is_packed(dev)))
1020                 return -1;
1021
1022         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1023                 return -1;
1024
1025         vq = dev->virtqueue[vring_idx];
1026         if (unlikely(!vq))
1027                 return -1;
1028
1029         if (unlikely(!vq->inflight_split))
1030                 return -1;
1031
1032         if (unlikely(idx >= vq->size))
1033                 return -1;
1034
1035         vq->inflight_split->desc[idx].counter = vq->global_counter++;
1036         vq->inflight_split->desc[idx].inflight = 1;
1037         return 0;
1038 }
1039
1040 int
1041 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx,
1042                                    uint16_t head, uint16_t last,
1043                                    uint16_t *inflight_entry)
1044 {
1045         struct rte_vhost_inflight_info_packed *inflight_info;
1046         struct virtio_net *dev;
1047         struct vhost_virtqueue *vq;
1048         struct vring_packed_desc *desc;
1049         uint16_t old_free_head, free_head;
1050
1051         dev = get_device(vid);
1052         if (unlikely(!dev))
1053                 return -1;
1054
1055         if (unlikely(!(dev->protocol_features &
1056             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1057                 return 0;
1058
1059         if (unlikely(!vq_is_packed(dev)))
1060                 return -1;
1061
1062         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1063                 return -1;
1064
1065         vq = dev->virtqueue[vring_idx];
1066         if (unlikely(!vq))
1067                 return -1;
1068
1069         inflight_info = vq->inflight_packed;
1070         if (unlikely(!inflight_info))
1071                 return -1;
1072
1073         if (unlikely(head >= vq->size))
1074                 return -1;
1075
1076         desc = vq->desc_packed;
1077         old_free_head = inflight_info->old_free_head;
1078         if (unlikely(old_free_head >= vq->size))
1079                 return -1;
1080
1081         free_head = old_free_head;
1082
1083         /* init header descriptor */
1084         inflight_info->desc[old_free_head].num = 0;
1085         inflight_info->desc[old_free_head].counter = vq->global_counter++;
1086         inflight_info->desc[old_free_head].inflight = 1;
1087
1088         /* save desc entry in flight entry */
1089         while (head != ((last + 1) % vq->size)) {
1090                 inflight_info->desc[old_free_head].num++;
1091                 inflight_info->desc[free_head].addr = desc[head].addr;
1092                 inflight_info->desc[free_head].len = desc[head].len;
1093                 inflight_info->desc[free_head].flags = desc[head].flags;
1094                 inflight_info->desc[free_head].id = desc[head].id;
1095
1096                 inflight_info->desc[old_free_head].last = free_head;
1097                 free_head = inflight_info->desc[free_head].next;
1098                 inflight_info->free_head = free_head;
1099                 head = (head + 1) % vq->size;
1100         }
1101
1102         inflight_info->old_free_head = free_head;
1103         *inflight_entry = old_free_head;
1104
1105         return 0;
1106 }
1107
1108 int
1109 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
1110                                   uint16_t last_used_idx, uint16_t idx)
1111 {
1112         struct virtio_net *dev;
1113         struct vhost_virtqueue *vq;
1114
1115         dev = get_device(vid);
1116         if (unlikely(!dev))
1117                 return -1;
1118
1119         if (unlikely(!(dev->protocol_features &
1120             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1121                 return 0;
1122
1123         if (unlikely(vq_is_packed(dev)))
1124                 return -1;
1125
1126         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1127                 return -1;
1128
1129         vq = dev->virtqueue[vring_idx];
1130         if (unlikely(!vq))
1131                 return -1;
1132
1133         if (unlikely(!vq->inflight_split))
1134                 return -1;
1135
1136         if (unlikely(idx >= vq->size))
1137                 return -1;
1138
1139         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1140
1141         vq->inflight_split->desc[idx].inflight = 0;
1142
1143         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1144
1145         vq->inflight_split->used_idx = last_used_idx;
1146         return 0;
1147 }
1148
1149 int
1150 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx,
1151                                    uint16_t head)
1152 {
1153         struct rte_vhost_inflight_info_packed *inflight_info;
1154         struct virtio_net *dev;
1155         struct vhost_virtqueue *vq;
1156
1157         dev = get_device(vid);
1158         if (unlikely(!dev))
1159                 return -1;
1160
1161         if (unlikely(!(dev->protocol_features &
1162             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1163                 return 0;
1164
1165         if (unlikely(!vq_is_packed(dev)))
1166                 return -1;
1167
1168         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1169                 return -1;
1170
1171         vq = dev->virtqueue[vring_idx];
1172         if (unlikely(!vq))
1173                 return -1;
1174
1175         inflight_info = vq->inflight_packed;
1176         if (unlikely(!inflight_info))
1177                 return -1;
1178
1179         if (unlikely(head >= vq->size))
1180                 return -1;
1181
1182         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1183
1184         inflight_info->desc[head].inflight = 0;
1185
1186         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1187
1188         inflight_info->old_free_head = inflight_info->free_head;
1189         inflight_info->old_used_idx = inflight_info->used_idx;
1190         inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter;
1191
1192         return 0;
1193 }
1194
1195 int
1196 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx,
1197                                      uint16_t idx)
1198 {
1199         struct virtio_net *dev;
1200         struct vhost_virtqueue *vq;
1201
1202         dev = get_device(vid);
1203         if (unlikely(!dev))
1204                 return -1;
1205
1206         if (unlikely(!(dev->protocol_features &
1207             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1208                 return 0;
1209
1210         if (unlikely(vq_is_packed(dev)))
1211                 return -1;
1212
1213         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1214                 return -1;
1215
1216         vq = dev->virtqueue[vring_idx];
1217         if (unlikely(!vq))
1218                 return -1;
1219
1220         if (unlikely(!vq->inflight_split))
1221                 return -1;
1222
1223         if (unlikely(idx >= vq->size))
1224                 return -1;
1225
1226         vq->inflight_split->last_inflight_io = idx;
1227         return 0;
1228 }
1229
1230 int
1231 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx,
1232                                       uint16_t head)
1233 {
1234         struct rte_vhost_inflight_info_packed *inflight_info;
1235         struct virtio_net *dev;
1236         struct vhost_virtqueue *vq;
1237         uint16_t last;
1238
1239         dev = get_device(vid);
1240         if (unlikely(!dev))
1241                 return -1;
1242
1243         if (unlikely(!(dev->protocol_features &
1244             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1245                 return 0;
1246
1247         if (unlikely(!vq_is_packed(dev)))
1248                 return -1;
1249
1250         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1251                 return -1;
1252
1253         vq = dev->virtqueue[vring_idx];
1254         if (unlikely(!vq))
1255                 return -1;
1256
1257         inflight_info = vq->inflight_packed;
1258         if (unlikely(!inflight_info))
1259                 return -1;
1260
1261         if (unlikely(head >= vq->size))
1262                 return -1;
1263
1264         last = inflight_info->desc[head].last;
1265         if (unlikely(last >= vq->size))
1266                 return -1;
1267
1268         inflight_info->desc[last].next = inflight_info->free_head;
1269         inflight_info->free_head = head;
1270         inflight_info->used_idx += inflight_info->desc[head].num;
1271         if (inflight_info->used_idx >= inflight_info->desc_num) {
1272                 inflight_info->used_idx -= inflight_info->desc_num;
1273                 inflight_info->used_wrap_counter =
1274                         !inflight_info->used_wrap_counter;
1275         }
1276
1277         return 0;
1278 }
1279
1280 int
1281 rte_vhost_vring_call(int vid, uint16_t vring_idx)
1282 {
1283         struct virtio_net *dev;
1284         struct vhost_virtqueue *vq;
1285
1286         dev = get_device(vid);
1287         if (!dev)
1288                 return -1;
1289
1290         if (vring_idx >= VHOST_MAX_VRING)
1291                 return -1;
1292
1293         vq = dev->virtqueue[vring_idx];
1294         if (!vq)
1295                 return -1;
1296
1297         if (vq_is_packed(dev))
1298                 vhost_vring_call_packed(dev, vq);
1299         else
1300                 vhost_vring_call_split(dev, vq);
1301
1302         return 0;
1303 }
1304
1305 uint16_t
1306 rte_vhost_avail_entries(int vid, uint16_t queue_id)
1307 {
1308         struct virtio_net *dev;
1309         struct vhost_virtqueue *vq;
1310         uint16_t ret = 0;
1311
1312         dev = get_device(vid);
1313         if (!dev)
1314                 return 0;
1315
1316         if (queue_id >= VHOST_MAX_VRING)
1317                 return 0;
1318
1319         vq = dev->virtqueue[queue_id];
1320         if (!vq)
1321                 return 0;
1322
1323         rte_spinlock_lock(&vq->access_lock);
1324
1325         if (unlikely(!vq->enabled || vq->avail == NULL))
1326                 goto out;
1327
1328         ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx;
1329
1330 out:
1331         rte_spinlock_unlock(&vq->access_lock);
1332         return ret;
1333 }
1334
1335 static inline int
1336 vhost_enable_notify_split(struct virtio_net *dev,
1337                 struct vhost_virtqueue *vq, int enable)
1338 {
1339         if (vq->used == NULL)
1340                 return -1;
1341
1342         if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
1343                 if (enable)
1344                         vq->used->flags &= ~VRING_USED_F_NO_NOTIFY;
1345                 else
1346                         vq->used->flags |= VRING_USED_F_NO_NOTIFY;
1347         } else {
1348                 if (enable)
1349                         vhost_avail_event(vq) = vq->last_avail_idx;
1350         }
1351         return 0;
1352 }
1353
1354 static inline int
1355 vhost_enable_notify_packed(struct virtio_net *dev,
1356                 struct vhost_virtqueue *vq, int enable)
1357 {
1358         uint16_t flags;
1359
1360         if (vq->device_event == NULL)
1361                 return -1;
1362
1363         if (!enable) {
1364                 vq->device_event->flags = VRING_EVENT_F_DISABLE;
1365                 return 0;
1366         }
1367
1368         flags = VRING_EVENT_F_ENABLE;
1369         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
1370                 flags = VRING_EVENT_F_DESC;
1371                 vq->device_event->off_wrap = vq->last_avail_idx |
1372                         vq->avail_wrap_counter << 15;
1373         }
1374
1375         rte_atomic_thread_fence(__ATOMIC_RELEASE);
1376
1377         vq->device_event->flags = flags;
1378         return 0;
1379 }
1380
1381 int
1382 vhost_enable_guest_notification(struct virtio_net *dev,
1383                 struct vhost_virtqueue *vq, int enable)
1384 {
1385         /*
1386          * If the virtqueue is not ready yet, it will be applied
1387          * when it will become ready.
1388          */
1389         if (!vq->ready)
1390                 return 0;
1391
1392         if (vq_is_packed(dev))
1393                 return vhost_enable_notify_packed(dev, vq, enable);
1394         else
1395                 return vhost_enable_notify_split(dev, vq, enable);
1396 }
1397
1398 int
1399 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
1400 {
1401         struct virtio_net *dev = get_device(vid);
1402         struct vhost_virtqueue *vq;
1403         int ret;
1404
1405         if (!dev)
1406                 return -1;
1407
1408         if (queue_id >= VHOST_MAX_VRING)
1409                 return -1;
1410
1411         vq = dev->virtqueue[queue_id];
1412         if (!vq)
1413                 return -1;
1414
1415         rte_spinlock_lock(&vq->access_lock);
1416
1417         vq->notif_enable = enable;
1418         ret = vhost_enable_guest_notification(dev, vq, enable);
1419
1420         rte_spinlock_unlock(&vq->access_lock);
1421
1422         return ret;
1423 }
1424
1425 void
1426 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len)
1427 {
1428         struct virtio_net *dev = get_device(vid);
1429
1430         if (dev == NULL)
1431                 return;
1432
1433         vhost_log_write(dev, addr, len);
1434 }
1435
1436 void
1437 rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
1438                          uint64_t offset, uint64_t len)
1439 {
1440         struct virtio_net *dev;
1441         struct vhost_virtqueue *vq;
1442
1443         dev = get_device(vid);
1444         if (dev == NULL)
1445                 return;
1446
1447         if (vring_idx >= VHOST_MAX_VRING)
1448                 return;
1449         vq = dev->virtqueue[vring_idx];
1450         if (!vq)
1451                 return;
1452
1453         vhost_log_used_vring(dev, vq, offset, len);
1454 }
1455
1456 uint32_t
1457 rte_vhost_rx_queue_count(int vid, uint16_t qid)
1458 {
1459         struct virtio_net *dev;
1460         struct vhost_virtqueue *vq;
1461         uint32_t ret = 0;
1462
1463         dev = get_device(vid);
1464         if (dev == NULL)
1465                 return 0;
1466
1467         if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) {
1468                 VHOST_LOG_DATA(ERR, "(%s) %s: invalid virtqueue idx %d.\n",
1469                         dev->ifname, __func__, qid);
1470                 return 0;
1471         }
1472
1473         vq = dev->virtqueue[qid];
1474         if (vq == NULL)
1475                 return 0;
1476
1477         rte_spinlock_lock(&vq->access_lock);
1478
1479         if (unlikely(!vq->enabled || vq->avail == NULL))
1480                 goto out;
1481
1482         ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx;
1483
1484 out:
1485         rte_spinlock_unlock(&vq->access_lock);
1486         return ret;
1487 }
1488
1489 struct rte_vdpa_device *
1490 rte_vhost_get_vdpa_device(int vid)
1491 {
1492         struct virtio_net *dev = get_device(vid);
1493
1494         if (dev == NULL)
1495                 return NULL;
1496
1497         return dev->vdpa_dev;
1498 }
1499
1500 int
1501 rte_vhost_get_log_base(int vid, uint64_t *log_base,
1502                 uint64_t *log_size)
1503 {
1504         struct virtio_net *dev = get_device(vid);
1505
1506         if (dev == NULL || log_base == NULL || log_size == NULL)
1507                 return -1;
1508
1509         *log_base = dev->log_base;
1510         *log_size = dev->log_size;
1511
1512         return 0;
1513 }
1514
1515 int
1516 rte_vhost_get_vring_base(int vid, uint16_t queue_id,
1517                 uint16_t *last_avail_idx, uint16_t *last_used_idx)
1518 {
1519         struct vhost_virtqueue *vq;
1520         struct virtio_net *dev = get_device(vid);
1521
1522         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1523                 return -1;
1524
1525         if (queue_id >= VHOST_MAX_VRING)
1526                 return -1;
1527
1528         vq = dev->virtqueue[queue_id];
1529         if (!vq)
1530                 return -1;
1531
1532         if (vq_is_packed(dev)) {
1533                 *last_avail_idx = (vq->avail_wrap_counter << 15) |
1534                                   vq->last_avail_idx;
1535                 *last_used_idx = (vq->used_wrap_counter << 15) |
1536                                  vq->last_used_idx;
1537         } else {
1538                 *last_avail_idx = vq->last_avail_idx;
1539                 *last_used_idx = vq->last_used_idx;
1540         }
1541
1542         return 0;
1543 }
1544
1545 int
1546 rte_vhost_set_vring_base(int vid, uint16_t queue_id,
1547                 uint16_t last_avail_idx, uint16_t last_used_idx)
1548 {
1549         struct vhost_virtqueue *vq;
1550         struct virtio_net *dev = get_device(vid);
1551
1552         if (!dev)
1553                 return -1;
1554
1555         if (queue_id >= VHOST_MAX_VRING)
1556                 return -1;
1557
1558         vq = dev->virtqueue[queue_id];
1559         if (!vq)
1560                 return -1;
1561
1562         if (vq_is_packed(dev)) {
1563                 vq->last_avail_idx = last_avail_idx & 0x7fff;
1564                 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15));
1565                 vq->last_used_idx = last_used_idx & 0x7fff;
1566                 vq->used_wrap_counter = !!(last_used_idx & (1 << 15));
1567         } else {
1568                 vq->last_avail_idx = last_avail_idx;
1569                 vq->last_used_idx = last_used_idx;
1570         }
1571
1572         return 0;
1573 }
1574
1575 int
1576 rte_vhost_get_vring_base_from_inflight(int vid,
1577                                        uint16_t queue_id,
1578                                        uint16_t *last_avail_idx,
1579                                        uint16_t *last_used_idx)
1580 {
1581         struct rte_vhost_inflight_info_packed *inflight_info;
1582         struct vhost_virtqueue *vq;
1583         struct virtio_net *dev = get_device(vid);
1584
1585         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1586                 return -1;
1587
1588         if (queue_id >= VHOST_MAX_VRING)
1589                 return -1;
1590
1591         vq = dev->virtqueue[queue_id];
1592         if (!vq)
1593                 return -1;
1594
1595         if (!vq_is_packed(dev))
1596                 return -1;
1597
1598         inflight_info = vq->inflight_packed;
1599         if (!inflight_info)
1600                 return -1;
1601
1602         *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) |
1603                           inflight_info->old_used_idx;
1604         *last_used_idx = *last_avail_idx;
1605
1606         return 0;
1607 }
1608
1609 int
1610 rte_vhost_extern_callback_register(int vid,
1611                 struct rte_vhost_user_extern_ops const * const ops, void *ctx)
1612 {
1613         struct virtio_net *dev = get_device(vid);
1614
1615         if (dev == NULL || ops == NULL)
1616                 return -1;
1617
1618         dev->extern_ops = *ops;
1619         dev->extern_data = ctx;
1620         return 0;
1621 }
1622
1623 static __rte_always_inline int
1624 async_channel_register(int vid, uint16_t queue_id,
1625                 struct rte_vhost_async_channel_ops *ops)
1626 {
1627         struct virtio_net *dev = get_device(vid);
1628         struct vhost_virtqueue *vq = dev->virtqueue[queue_id];
1629         struct vhost_async *async;
1630         int node = vq->numa_node;
1631
1632         if (unlikely(vq->async)) {
1633                 VHOST_LOG_CONFIG(ERR,
1634                                 "(%s) async register failed: already registered (qid: %d)\n",
1635                                 dev->ifname, queue_id);
1636                 return -1;
1637         }
1638
1639         async = rte_zmalloc_socket(NULL, sizeof(struct vhost_async), 0, node);
1640         if (!async) {
1641                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async metadata (qid: %d)\n",
1642                                 dev->ifname, queue_id);
1643                 return -1;
1644         }
1645
1646         async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info),
1647                         RTE_CACHE_LINE_SIZE, node);
1648         if (!async->pkts_info) {
1649                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async_pkts_info (qid: %d)\n",
1650                                 dev->ifname, queue_id);
1651                 goto out_free_async;
1652         }
1653
1654         if (vq_is_packed(dev)) {
1655                 async->buffers_packed = rte_malloc_socket(NULL,
1656                                 vq->size * sizeof(struct vring_used_elem_packed),
1657                                 RTE_CACHE_LINE_SIZE, node);
1658                 if (!async->buffers_packed) {
1659                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async buffers (qid: %d)\n",
1660                                         dev->ifname, queue_id);
1661                         goto out_free_inflight;
1662                 }
1663         } else {
1664                 async->descs_split = rte_malloc_socket(NULL,
1665                                 vq->size * sizeof(struct vring_used_elem),
1666                                 RTE_CACHE_LINE_SIZE, node);
1667                 if (!async->descs_split) {
1668                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async descs (qid: %d)\n",
1669                                         dev->ifname, queue_id);
1670                         goto out_free_inflight;
1671                 }
1672         }
1673
1674         async->ops.check_completed_copies = ops->check_completed_copies;
1675         async->ops.transfer_data = ops->transfer_data;
1676
1677         vq->async = async;
1678
1679         return 0;
1680 out_free_inflight:
1681         rte_free(async->pkts_info);
1682 out_free_async:
1683         rte_free(async);
1684
1685         return -1;
1686 }
1687
1688 int
1689 rte_vhost_async_channel_register(int vid, uint16_t queue_id,
1690                 struct rte_vhost_async_config config,
1691                 struct rte_vhost_async_channel_ops *ops)
1692 {
1693         struct vhost_virtqueue *vq;
1694         struct virtio_net *dev = get_device(vid);
1695         int ret;
1696
1697         if (dev == NULL || ops == NULL)
1698                 return -1;
1699
1700         if (queue_id >= VHOST_MAX_VRING)
1701                 return -1;
1702
1703         vq = dev->virtqueue[queue_id];
1704
1705         if (unlikely(vq == NULL || !dev->async_copy))
1706                 return -1;
1707
1708         if (unlikely(!(config.features & RTE_VHOST_ASYNC_INORDER))) {
1709                 VHOST_LOG_CONFIG(ERR,
1710                         "(%s) async copy is not supported on non-inorder mode (qid: %d)\n",
1711                         dev->ifname, queue_id);
1712                 return -1;
1713         }
1714
1715         if (unlikely(ops->check_completed_copies == NULL ||
1716                 ops->transfer_data == NULL))
1717                 return -1;
1718
1719         rte_spinlock_lock(&vq->access_lock);
1720         ret = async_channel_register(vid, queue_id, ops);
1721         rte_spinlock_unlock(&vq->access_lock);
1722
1723         return ret;
1724 }
1725
1726 int
1727 rte_vhost_async_channel_register_thread_unsafe(int vid, uint16_t queue_id,
1728                 struct rte_vhost_async_config config,
1729                 struct rte_vhost_async_channel_ops *ops)
1730 {
1731         struct vhost_virtqueue *vq;
1732         struct virtio_net *dev = get_device(vid);
1733
1734         if (dev == NULL || ops == NULL)
1735                 return -1;
1736
1737         if (queue_id >= VHOST_MAX_VRING)
1738                 return -1;
1739
1740         vq = dev->virtqueue[queue_id];
1741
1742         if (unlikely(vq == NULL || !dev->async_copy))
1743                 return -1;
1744
1745         if (unlikely(!(config.features & RTE_VHOST_ASYNC_INORDER))) {
1746                 VHOST_LOG_CONFIG(ERR,
1747                         "(%s) async copy is not supported on non-inorder mode (qid: %d)\n",
1748                         dev->ifname, queue_id);
1749                 return -1;
1750         }
1751
1752         if (unlikely(ops->check_completed_copies == NULL ||
1753                 ops->transfer_data == NULL))
1754                 return -1;
1755
1756         return async_channel_register(vid, queue_id, ops);
1757 }
1758
1759 int
1760 rte_vhost_async_channel_unregister(int vid, uint16_t queue_id)
1761 {
1762         struct vhost_virtqueue *vq;
1763         struct virtio_net *dev = get_device(vid);
1764         int ret = -1;
1765
1766         if (dev == NULL)
1767                 return ret;
1768
1769         if (queue_id >= VHOST_MAX_VRING)
1770                 return ret;
1771
1772         vq = dev->virtqueue[queue_id];
1773
1774         if (vq == NULL)
1775                 return ret;
1776
1777         ret = 0;
1778
1779         if (!vq->async)
1780                 return ret;
1781
1782         if (!rte_spinlock_trylock(&vq->access_lock)) {
1783                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel, virtqueue busy.\n",
1784                                 dev->ifname);
1785                 return -1;
1786         }
1787
1788         if (vq->async->pkts_inflight_n) {
1789                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname);
1790                 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n",
1791                         dev->ifname);
1792                 ret = -1;
1793                 goto out;
1794         }
1795
1796         vhost_free_async_mem(vq);
1797 out:
1798         rte_spinlock_unlock(&vq->access_lock);
1799
1800         return ret;
1801 }
1802
1803 int
1804 rte_vhost_async_channel_unregister_thread_unsafe(int vid, uint16_t queue_id)
1805 {
1806         struct vhost_virtqueue *vq;
1807         struct virtio_net *dev = get_device(vid);
1808
1809         if (dev == NULL)
1810                 return -1;
1811
1812         if (queue_id >= VHOST_MAX_VRING)
1813                 return -1;
1814
1815         vq = dev->virtqueue[queue_id];
1816
1817         if (vq == NULL)
1818                 return -1;
1819
1820         if (!vq->async)
1821                 return 0;
1822
1823         if (vq->async->pkts_inflight_n) {
1824                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname);
1825                 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n",
1826                         dev->ifname);
1827                 return -1;
1828         }
1829
1830         vhost_free_async_mem(vq);
1831
1832         return 0;
1833 }
1834
1835 int
1836 rte_vhost_async_get_inflight(int vid, uint16_t queue_id)
1837 {
1838         struct vhost_virtqueue *vq;
1839         struct virtio_net *dev = get_device(vid);
1840         int ret = -1;
1841
1842         if (dev == NULL)
1843                 return ret;
1844
1845         if (queue_id >= VHOST_MAX_VRING)
1846                 return ret;
1847
1848         vq = dev->virtqueue[queue_id];
1849
1850         if (vq == NULL)
1851                 return ret;
1852
1853         if (!vq->async)
1854                 return ret;
1855
1856         if (!rte_spinlock_trylock(&vq->access_lock)) {
1857                 VHOST_LOG_CONFIG(DEBUG,
1858                         "(%s) failed to check in-flight packets. virtqueue busy.\n",
1859                         dev->ifname);
1860                 return ret;
1861         }
1862
1863         ret = vq->async->pkts_inflight_n;
1864         rte_spinlock_unlock(&vq->access_lock);
1865
1866         return ret;
1867 }
1868
1869 int
1870 rte_vhost_get_monitor_addr(int vid, uint16_t queue_id,
1871                 struct rte_vhost_power_monitor_cond *pmc)
1872 {
1873         struct virtio_net *dev = get_device(vid);
1874         struct vhost_virtqueue *vq;
1875
1876         if (dev == NULL)
1877                 return -1;
1878         if (queue_id >= VHOST_MAX_VRING)
1879                 return -1;
1880
1881         vq = dev->virtqueue[queue_id];
1882         if (vq == NULL)
1883                 return -1;
1884
1885         if (vq_is_packed(dev)) {
1886                 struct vring_packed_desc *desc;
1887                 desc = vq->desc_packed;
1888                 pmc->addr = &desc[vq->last_avail_idx].flags;
1889                 if (vq->avail_wrap_counter)
1890                         pmc->val = VRING_DESC_F_AVAIL;
1891                 else
1892                         pmc->val = VRING_DESC_F_USED;
1893                 pmc->mask = VRING_DESC_F_AVAIL | VRING_DESC_F_USED;
1894                 pmc->size = sizeof(desc[vq->last_avail_idx].flags);
1895                 pmc->match = 1;
1896         } else {
1897                 pmc->addr = &vq->avail->idx;
1898                 pmc->val = vq->last_avail_idx & (vq->size - 1);
1899                 pmc->mask = vq->size - 1;
1900                 pmc->size = sizeof(vq->avail->idx);
1901                 pmc->match = 0;
1902         }
1903
1904         return 0;
1905 }
1906
1907 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO);
1908 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING);