lib/vhost/vhost.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <linux/vhost.h>
   6 #include <linux/virtio_net.h>
   7 #include <stdint.h>
   8 #include <stdlib.h>
   9 #ifdef RTE_LIBRTE_VHOST_NUMA
  10 #include <numa.h>
  11 #include <numaif.h>
  12 #endif
  13
  14 #include <rte_errno.h>
  15 #include <rte_log.h>
  16 #include <rte_memory.h>
  17 #include <rte_malloc.h>
  18 #include <rte_vhost.h>
  19
  20 #include "iotlb.h"
  21 #include "vhost.h"
  22 #include "vhost_user.h"
  23
  24 struct virtio_net *vhost_devices[RTE_MAX_VHOST_DEVICE];
  25 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER;
  26
  27 /* Called with iotlb_lock read-locked */
  28 uint64_t
  29 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
  30                     uint64_t iova, uint64_t *size, uint8_t perm)
  31 {
  32         uint64_t vva, tmp_size;
  33
  34         if (unlikely(!*size))
  35                 return 0;
  36
  37         tmp_size = *size;
  38
  39         vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm);
  40         if (tmp_size == *size)
  41                 return vva;
  42
  43         iova += tmp_size;
  44
  45         if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) {
  46                 /*
  47                  * iotlb_lock is read-locked for a full burst,
  48                  * but it only protects the iotlb cache.
  49                  * In case of IOTLB miss, we might block on the socket,
  50                  * which could cause a deadlock with QEMU if an IOTLB update
  51                  * is being handled. We can safely unlock here to avoid it.
  52                  */
  53                 vhost_user_iotlb_rd_unlock(vq);
  54
  55                 vhost_user_iotlb_pending_insert(dev, vq, iova, perm);
  56                 if (vhost_user_iotlb_miss(dev, iova, perm)) {
  57                         VHOST_LOG_DATA(ERR, "(%s) IOTLB miss req failed for IOVA 0x%" PRIx64 "\n",
  58                                 dev->ifname, iova);
  59                         vhost_user_iotlb_pending_remove(vq, iova, 1, perm);
  60                 }
  61
  62                 vhost_user_iotlb_rd_lock(vq);
  63         }
  64
  65         return 0;
  66 }
  67
  68 #define VHOST_LOG_PAGE  4096
  69
  70 /*
  71  * Atomically set a bit in memory.
  72  */
  73 static __rte_always_inline void
  74 vhost_set_bit(unsigned int nr, volatile uint8_t *addr)
  75 {
  76 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
  77         /*
  78          * __sync_ built-ins are deprecated, but __atomic_ ones
  79          * are sub-optimized in older GCC versions.
  80          */
  81         __sync_fetch_and_or_1(addr, (1U << nr));
  82 #else
  83         __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED);
  84 #endif
  85 }
  86
  87 static __rte_always_inline void
  88 vhost_log_page(uint8_t *log_base, uint64_t page)
  89 {
  90         vhost_set_bit(page % 8, &log_base[page / 8]);
  91 }
  92
  93 void
  94 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
  95 {
  96         uint64_t page;
  97
  98         if (unlikely(!dev->log_base || !len))
  99                 return;
 100
 101         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 102                 return;
 103
 104         /* To make sure guest memory updates are committed before logging */
 105         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 106
 107         page = addr / VHOST_LOG_PAGE;
 108         while (page * VHOST_LOG_PAGE < addr + len) {
 109                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 110                 page += 1;
 111         }
 112 }
 113
 114 void
 115 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 116                              uint64_t iova, uint64_t len)
 117 {
 118         uint64_t hva, gpa, map_len;
 119         map_len = len;
 120
 121         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 122         if (map_len != len) {
 123                 VHOST_LOG_DATA(ERR,
 124                         "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 125                         dev->ifname, iova);
 126                 return;
 127         }
 128
 129         gpa = hva_to_gpa(dev, hva, len);
 130         if (gpa)
 131                 __vhost_log_write(dev, gpa, len);
 132 }
 133
 134 void
 135 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
 136 {
 137         unsigned long *log_base;
 138         int i;
 139
 140         if (unlikely(!dev->log_base))
 141                 return;
 142
 143         /* No cache, nothing to sync */
 144         if (unlikely(!vq->log_cache))
 145                 return;
 146
 147         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 148
 149         log_base = (unsigned long *)(uintptr_t)dev->log_base;
 150
 151         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 152                 struct log_cache_entry *elem = vq->log_cache + i;
 153
 154 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
 155                 /*
 156                  * '__sync' builtins are deprecated, but '__atomic' ones
 157                  * are sub-optimized in older GCC versions.
 158                  */
 159                 __sync_fetch_and_or(log_base + elem->offset, elem->val);
 160 #else
 161                 __atomic_fetch_or(log_base + elem->offset, elem->val,
 162                                 __ATOMIC_RELAXED);
 163 #endif
 164         }
 165
 166         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 167
 168         vq->log_cache_nb_elem = 0;
 169 }
 170
 171 static __rte_always_inline void
 172 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
 173                         uint64_t page)
 174 {
 175         uint32_t bit_nr = page % (sizeof(unsigned long) << 3);
 176         uint32_t offset = page / (sizeof(unsigned long) << 3);
 177         int i;
 178
 179         if (unlikely(!vq->log_cache)) {
 180                 /* No logging cache allocated, write dirty log map directly */
 181                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 182                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 183
 184                 return;
 185         }
 186
 187         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 188                 struct log_cache_entry *elem = vq->log_cache + i;
 189
 190                 if (elem->offset == offset) {
 191                         elem->val |= (1UL << bit_nr);
 192                         return;
 193                 }
 194         }
 195
 196         if (unlikely(i >= VHOST_LOG_CACHE_NR)) {
 197                 /*
 198                  * No more room for a new log cache entry,
 199                  * so write the dirty log map directly.
 200                  */
 201                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 202                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 203
 204                 return;
 205         }
 206
 207         vq->log_cache[i].offset = offset;
 208         vq->log_cache[i].val = (1UL << bit_nr);
 209         vq->log_cache_nb_elem++;
 210 }
 211
 212 void
 213 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
 214                         uint64_t addr, uint64_t len)
 215 {
 216         uint64_t page;
 217
 218         if (unlikely(!dev->log_base || !len))
 219                 return;
 220
 221         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 222                 return;
 223
 224         page = addr / VHOST_LOG_PAGE;
 225         while (page * VHOST_LOG_PAGE < addr + len) {
 226                 vhost_log_cache_page(dev, vq, page);
 227                 page += 1;
 228         }
 229 }
 230
 231 void
 232 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 233                              uint64_t iova, uint64_t len)
 234 {
 235         uint64_t hva, gpa, map_len;
 236         map_len = len;
 237
 238         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 239         if (map_len != len) {
 240                 VHOST_LOG_DATA(ERR,
 241                         "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 242                         dev->ifname, iova);
 243                 return;
 244         }
 245
 246         gpa = hva_to_gpa(dev, hva, len);
 247         if (gpa)
 248                 __vhost_log_cache_write(dev, vq, gpa, len);
 249 }
 250
 251 void *
 252 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
 253                 uint64_t desc_addr, uint64_t desc_len)
 254 {
 255         void *idesc;
 256         uint64_t src, dst;
 257         uint64_t len, remain = desc_len;
 258
 259         idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node);
 260         if (unlikely(!idesc))
 261                 return NULL;
 262
 263         dst = (uint64_t)(uintptr_t)idesc;
 264
 265         while (remain) {
 266                 len = remain;
 267                 src = vhost_iova_to_vva(dev, vq, desc_addr, &len,
 268                                 VHOST_ACCESS_RO);
 269                 if (unlikely(!src || !len)) {
 270                         rte_free(idesc);
 271                         return NULL;
 272                 }
 273
 274                 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len);
 275
 276                 remain -= len;
 277                 dst += len;
 278                 desc_addr += len;
 279         }
 280
 281         return idesc;
 282 }
 283
 284 void
 285 cleanup_vq(struct vhost_virtqueue *vq, int destroy)
 286 {
 287         if ((vq->callfd >= 0) && (destroy != 0))
 288                 close(vq->callfd);
 289         if (vq->kickfd >= 0)
 290                 close(vq->kickfd);
 291 }
 292
 293 void
 294 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq)
 295 {
 296         if (!(dev->protocol_features &
 297             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
 298                 return;
 299
 300         if (vq_is_packed(dev)) {
 301                 if (vq->inflight_packed)
 302                         vq->inflight_packed = NULL;
 303         } else {
 304                 if (vq->inflight_split)
 305                         vq->inflight_split = NULL;
 306         }
 307
 308         if (vq->resubmit_inflight) {
 309                 if (vq->resubmit_inflight->resubmit_list) {
 310                         rte_free(vq->resubmit_inflight->resubmit_list);
 311                         vq->resubmit_inflight->resubmit_list = NULL;
 312                 }
 313                 rte_free(vq->resubmit_inflight);
 314                 vq->resubmit_inflight = NULL;
 315         }
 316 }
 317
 318 /*
 319  * Unmap any memory, close any file descriptors and
 320  * free any memory owned by a device.
 321  */
 322 void
 323 cleanup_device(struct virtio_net *dev, int destroy)
 324 {
 325         uint32_t i;
 326
 327         vhost_backend_cleanup(dev);
 328
 329         for (i = 0; i < dev->nr_vring; i++) {
 330                 cleanup_vq(dev->virtqueue[i], destroy);
 331                 cleanup_vq_inflight(dev, dev->virtqueue[i]);
 332         }
 333 }
 334
 335 static void
 336 vhost_free_async_mem(struct vhost_virtqueue *vq)
 337 {
 338         if (!vq->async)
 339                 return;
 340
 341         rte_free(vq->async->pkts_info);
 342         rte_free(vq->async->pkts_cmpl_flag);
 343
 344         rte_free(vq->async->buffers_packed);
 345         vq->async->buffers_packed = NULL;
 346         rte_free(vq->async->descs_split);
 347         vq->async->descs_split = NULL;
 348
 349         rte_free(vq->async);
 350         vq->async = NULL;
 351 }
 352
 353 void
 354 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 355 {
 356         if (vq_is_packed(dev))
 357                 rte_free(vq->shadow_used_packed);
 358         else
 359                 rte_free(vq->shadow_used_split);
 360
 361         vhost_free_async_mem(vq);
 362         rte_free(vq->batch_copy_elems);
 363         rte_mempool_free(vq->iotlb_pool);
 364         rte_free(vq->log_cache);
 365         rte_free(vq);
 366 }
 367
 368 /*
 369  * Release virtqueues and device memory.
 370  */
 371 static void
 372 free_device(struct virtio_net *dev)
 373 {
 374         uint32_t i;
 375
 376         for (i = 0; i < dev->nr_vring; i++)
 377                 free_vq(dev, dev->virtqueue[i]);
 378
 379         rte_free(dev);
 380 }
 381
 382 static __rte_always_inline int
 383 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 384 {
 385         if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG))))
 386                 return 0;
 387
 388         vq->log_guest_addr = translate_log_addr(dev, vq,
 389                                                 vq->ring_addrs.log_guest_addr);
 390         if (vq->log_guest_addr == 0)
 391                 return -1;
 392
 393         return 0;
 394 }
 395
 396 /*
 397  * Converts vring log address to GPA
 398  * If IOMMU is enabled, the log address is IOVA
 399  * If IOMMU not enabled, the log address is already GPA
 400  *
 401  * Caller should have iotlb_lock read-locked
 402  */
 403 uint64_t
 404 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 405                 uint64_t log_addr)
 406 {
 407         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 408                 const uint64_t exp_size = sizeof(uint64_t);
 409                 uint64_t hva, gpa;
 410                 uint64_t size = exp_size;
 411
 412                 hva = vhost_iova_to_vva(dev, vq, log_addr,
 413                                         &size, VHOST_ACCESS_RW);
 414
 415                 if (size != exp_size)
 416                         return 0;
 417
 418                 gpa = hva_to_gpa(dev, hva, exp_size);
 419                 if (!gpa) {
 420                         VHOST_LOG_DATA(ERR,
 421                                 "(%s) failed to find GPA for log_addr: 0x%"
 422                                 PRIx64 " hva: 0x%" PRIx64 "\n",
 423                                 dev->ifname, log_addr, hva);
 424                         return 0;
 425                 }
 426                 return gpa;
 427
 428         } else
 429                 return log_addr;
 430 }
 431
 432 /* Caller should have iotlb_lock read-locked */
 433 static int
 434 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 435 {
 436         uint64_t req_size, size;
 437
 438         req_size = sizeof(struct vring_desc) * vq->size;
 439         size = req_size;
 440         vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 441                                                 vq->ring_addrs.desc_user_addr,
 442                                                 &size, VHOST_ACCESS_RW);
 443         if (!vq->desc || size != req_size)
 444                 return -1;
 445
 446         req_size = sizeof(struct vring_avail);
 447         req_size += sizeof(uint16_t) * vq->size;
 448         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 449                 req_size += sizeof(uint16_t);
 450         size = req_size;
 451         vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 452                                                 vq->ring_addrs.avail_user_addr,
 453                                                 &size, VHOST_ACCESS_RW);
 454         if (!vq->avail || size != req_size)
 455                 return -1;
 456
 457         req_size = sizeof(struct vring_used);
 458         req_size += sizeof(struct vring_used_elem) * vq->size;
 459         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 460                 req_size += sizeof(uint16_t);
 461         size = req_size;
 462         vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 463                                                 vq->ring_addrs.used_user_addr,
 464                                                 &size, VHOST_ACCESS_RW);
 465         if (!vq->used || size != req_size)
 466                 return -1;
 467
 468         return 0;
 469 }
 470
 471 /* Caller should have iotlb_lock read-locked */
 472 static int
 473 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)
 474 {
 475         uint64_t req_size, size;
 476
 477         req_size = sizeof(struct vring_packed_desc) * vq->size;
 478         size = req_size;
 479         vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 480                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr,
 481                                 &size, VHOST_ACCESS_RW);
 482         if (!vq->desc_packed || size != req_size)
 483                 return -1;
 484
 485         req_size = sizeof(struct vring_packed_desc_event);
 486         size = req_size;
 487         vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t)
 488                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr,
 489                                 &size, VHOST_ACCESS_RW);
 490         if (!vq->driver_event || size != req_size)
 491                 return -1;
 492
 493         req_size = sizeof(struct vring_packed_desc_event);
 494         size = req_size;
 495         vq->device_event = (struct vring_packed_desc_event *)(uintptr_t)
 496                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr,
 497                                 &size, VHOST_ACCESS_RW);
 498         if (!vq->device_event || size != req_size)
 499                 return -1;
 500
 501         return 0;
 502 }
 503
 504 int
 505 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 506 {
 507
 508         if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
 509                 return -1;
 510
 511         if (vq_is_packed(dev)) {
 512                 if (vring_translate_packed(dev, vq) < 0)
 513                         return -1;
 514         } else {
 515                 if (vring_translate_split(dev, vq) < 0)
 516                         return -1;
 517         }
 518
 519         if (log_translate(dev, vq) < 0)
 520                 return -1;
 521
 522         vq->access_ok = true;
 523
 524         return 0;
 525 }
 526
 527 void
 528 vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 529 {
 530         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 531                 vhost_user_iotlb_wr_lock(vq);
 532
 533         vq->access_ok = false;
 534         vq->desc = NULL;
 535         vq->avail = NULL;
 536         vq->used = NULL;
 537         vq->log_guest_addr = 0;
 538
 539         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 540                 vhost_user_iotlb_wr_unlock(vq);
 541 }
 542
 543 static void
 544 init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 545 {
 546         struct vhost_virtqueue *vq;
 547         int numa_node = SOCKET_ID_ANY;
 548
 549         if (vring_idx >= VHOST_MAX_VRING) {
 550                 VHOST_LOG_CONFIG(ERR, "(%s) failed to init vring, out of bound (%d)\n",
 551                                 dev->ifname, vring_idx);
 552                 return;
 553         }
 554
 555         vq = dev->virtqueue[vring_idx];
 556         if (!vq) {
 557                 VHOST_LOG_CONFIG(ERR, "(%s) virtqueue not allocated (%d)\n",
 558                                 dev->ifname, vring_idx);
 559                 return;
 560         }
 561
 562         memset(vq, 0, sizeof(struct vhost_virtqueue));
 563
 564         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
 565         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
 566         vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
 567
 568 #ifdef RTE_LIBRTE_VHOST_NUMA
 569         if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) {
 570                 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n",
 571                         dev->ifname, rte_strerror(errno));
 572                 numa_node = SOCKET_ID_ANY;
 573         }
 574 #endif
 575         vq->numa_node = numa_node;
 576
 577         vhost_user_iotlb_init(dev, vring_idx);
 578 }
 579
 580 static void
 581 reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 582 {
 583         struct vhost_virtqueue *vq;
 584         int callfd;
 585
 586         if (vring_idx >= VHOST_MAX_VRING) {
 587                 VHOST_LOG_CONFIG(ERR,
 588                                 "(%s) failed to reset vring, out of bound (%d)\n",
 589                                 dev->ifname, vring_idx);
 590                 return;
 591         }
 592
 593         vq = dev->virtqueue[vring_idx];
 594         if (!vq) {
 595                 VHOST_LOG_CONFIG(ERR, "(%s) failed to reset vring, virtqueue not allocated (%d)\n",
 596                                 dev->ifname, vring_idx);
 597                 return;
 598         }
 599
 600         callfd = vq->callfd;
 601         init_vring_queue(dev, vring_idx);
 602         vq->callfd = callfd;
 603 }
 604
 605 int
 606 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 607 {
 608         struct vhost_virtqueue *vq;
 609         uint32_t i;
 610
 611         /* Also allocate holes, if any, up to requested vring index. */
 612         for (i = 0; i <= vring_idx; i++) {
 613                 if (dev->virtqueue[i])
 614                         continue;
 615
 616                 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0);
 617                 if (vq == NULL) {
 618                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate memory for vring %u.\n",
 619                                         dev->ifname, i);
 620                         return -1;
 621                 }
 622
 623                 dev->virtqueue[i] = vq;
 624                 init_vring_queue(dev, i);
 625                 rte_spinlock_init(&vq->access_lock);
 626                 vq->avail_wrap_counter = 1;
 627                 vq->used_wrap_counter = 1;
 628                 vq->signalled_used_valid = false;
 629         }
 630
 631         dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1);
 632
 633         return 0;
 634 }
 635
 636 /*
 637  * Reset some variables in device structure, while keeping few
 638  * others untouched, such as vid, ifname, nr_vring: they
 639  * should be same unless the device is removed.
 640  */
 641 void
 642 reset_device(struct virtio_net *dev)
 643 {
 644         uint32_t i;
 645
 646         dev->features = 0;
 647         dev->protocol_features = 0;
 648         dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 649
 650         for (i = 0; i < dev->nr_vring; i++)
 651                 reset_vring_queue(dev, i);
 652 }
 653
 654 /*
 655  * Invoked when there is a new vhost-user connection established (when
 656  * there is a new virtio device being attached).
 657  */
 658 int
 659 vhost_new_device(void)
 660 {
 661         struct virtio_net *dev;
 662         int i;
 663
 664         pthread_mutex_lock(&vhost_dev_lock);
 665         for (i = 0; i < RTE_MAX_VHOST_DEVICE; i++) {
 666                 if (vhost_devices[i] == NULL)
 667                         break;
 668         }
 669
 670         if (i == RTE_MAX_VHOST_DEVICE) {
 671                 VHOST_LOG_CONFIG(ERR, "failed to find a free slot for new device.\n");
 672                 pthread_mutex_unlock(&vhost_dev_lock);
 673                 return -1;
 674         }
 675
 676         dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0);
 677         if (dev == NULL) {
 678                 VHOST_LOG_CONFIG(ERR, "failed to allocate memory for new device.\n");
 679                 pthread_mutex_unlock(&vhost_dev_lock);
 680                 return -1;
 681         }
 682
 683         vhost_devices[i] = dev;
 684         pthread_mutex_unlock(&vhost_dev_lock);
 685
 686         dev->vid = i;
 687         dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 688         dev->slave_req_fd = -1;
 689         dev->postcopy_ufd = -1;
 690         rte_spinlock_init(&dev->slave_req_lock);
 691
 692         return i;
 693 }
 694
 695 void
 696 vhost_destroy_device_notify(struct virtio_net *dev)
 697 {
 698         struct rte_vdpa_device *vdpa_dev;
 699
 700         if (dev->flags & VIRTIO_DEV_RUNNING) {
 701                 vdpa_dev = dev->vdpa_dev;
 702                 if (vdpa_dev)
 703                         vdpa_dev->ops->dev_close(dev->vid);
 704                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 705                 dev->notify_ops->destroy_device(dev->vid);
 706         }
 707 }
 708
 709 /*
 710  * Invoked when there is the vhost-user connection is broken (when
 711  * the virtio device is being detached).
 712  */
 713 void
 714 vhost_destroy_device(int vid)
 715 {
 716         struct virtio_net *dev = get_device(vid);
 717
 718         if (dev == NULL)
 719                 return;
 720
 721         vhost_destroy_device_notify(dev);
 722
 723         cleanup_device(dev, 1);
 724         free_device(dev);
 725
 726         vhost_devices[vid] = NULL;
 727 }
 728
 729 void
 730 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev)
 731 {
 732         struct virtio_net *dev = get_device(vid);
 733
 734         if (dev == NULL)
 735                 return;
 736
 737         dev->vdpa_dev = vdpa_dev;
 738 }
 739
 740 void
 741 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len)
 742 {
 743         struct virtio_net *dev;
 744         unsigned int len;
 745
 746         dev = get_device(vid);
 747         if (dev == NULL)
 748                 return;
 749
 750         len = if_len > sizeof(dev->ifname) ?
 751                 sizeof(dev->ifname) : if_len;
 752
 753         strncpy(dev->ifname, if_name, len);
 754         dev->ifname[sizeof(dev->ifname) - 1] = '\0';
 755 }
 756
 757 void
 758 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags)
 759 {
 760         struct virtio_net *dev = get_device(vid);
 761
 762         if (dev == NULL)
 763                 return;
 764
 765         if (enable)
 766                 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 767         else
 768                 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 769         if (!compliant_ol_flags)
 770                 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS;
 771         else
 772                 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS;
 773 }
 774
 775 void
 776 vhost_enable_extbuf(int vid)
 777 {
 778         struct virtio_net *dev = get_device(vid);
 779
 780         if (dev == NULL)
 781                 return;
 782
 783         dev->extbuf = 1;
 784 }
 785
 786 void
 787 vhost_enable_linearbuf(int vid)
 788 {
 789         struct virtio_net *dev = get_device(vid);
 790
 791         if (dev == NULL)
 792                 return;
 793
 794         dev->linearbuf = 1;
 795 }
 796
 797 int
 798 rte_vhost_get_mtu(int vid, uint16_t *mtu)
 799 {
 800         struct virtio_net *dev = get_device(vid);
 801
 802         if (dev == NULL || mtu == NULL)
 803                 return -ENODEV;
 804
 805         if (!(dev->flags & VIRTIO_DEV_READY))
 806                 return -EAGAIN;
 807
 808         if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU)))
 809                 return -ENOTSUP;
 810
 811         *mtu = dev->mtu;
 812
 813         return 0;
 814 }
 815
 816 int
 817 rte_vhost_get_numa_node(int vid)
 818 {
 819 #ifdef RTE_LIBRTE_VHOST_NUMA
 820         struct virtio_net *dev = get_device(vid);
 821         int numa_node;
 822         int ret;
 823
 824         if (dev == NULL || numa_available() != 0)
 825                 return -1;
 826
 827         ret = get_mempolicy(&numa_node, NULL, 0, dev,
 828                             MPOL_F_NODE | MPOL_F_ADDR);
 829         if (ret < 0) {
 830                 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n",
 831                         dev->ifname, rte_strerror(errno));
 832                 return -1;
 833         }
 834
 835         return numa_node;
 836 #else
 837         RTE_SET_USED(vid);
 838         return -1;
 839 #endif
 840 }
 841
 842 uint32_t
 843 rte_vhost_get_queue_num(int vid)
 844 {
 845         struct virtio_net *dev = get_device(vid);
 846
 847         if (dev == NULL)
 848                 return 0;
 849
 850         return dev->nr_vring / 2;
 851 }
 852
 853 uint16_t
 854 rte_vhost_get_vring_num(int vid)
 855 {
 856         struct virtio_net *dev = get_device(vid);
 857
 858         if (dev == NULL)
 859                 return 0;
 860
 861         return dev->nr_vring;
 862 }
 863
 864 int
 865 rte_vhost_get_ifname(int vid, char *buf, size_t len)
 866 {
 867         struct virtio_net *dev = get_device(vid);
 868
 869         if (dev == NULL || buf == NULL)
 870                 return -1;
 871
 872         len = RTE_MIN(len, sizeof(dev->ifname));
 873
 874         strncpy(buf, dev->ifname, len);
 875         buf[len - 1] = '\0';
 876
 877         return 0;
 878 }
 879
 880 int
 881 rte_vhost_get_negotiated_features(int vid, uint64_t *features)
 882 {
 883         struct virtio_net *dev;
 884
 885         dev = get_device(vid);
 886         if (dev == NULL || features == NULL)
 887                 return -1;
 888
 889         *features = dev->features;
 890         return 0;
 891 }
 892
 893 int
 894 rte_vhost_get_negotiated_protocol_features(int vid,
 895                                            uint64_t *protocol_features)
 896 {
 897         struct virtio_net *dev;
 898
 899         dev = get_device(vid);
 900         if (dev == NULL || protocol_features == NULL)
 901                 return -1;
 902
 903         *protocol_features = dev->protocol_features;
 904         return 0;
 905 }
 906
 907 int
 908 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem)
 909 {
 910         struct virtio_net *dev;
 911         struct rte_vhost_memory *m;
 912         size_t size;
 913
 914         dev = get_device(vid);
 915         if (dev == NULL || mem == NULL)
 916                 return -1;
 917
 918         size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region);
 919         m = malloc(sizeof(struct rte_vhost_memory) + size);
 920         if (!m)
 921                 return -1;
 922
 923         m->nregions = dev->mem->nregions;
 924         memcpy(m->regions, dev->mem->regions, size);
 925         *mem = m;
 926
 927         return 0;
 928 }
 929
 930 int
 931 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
 932                           struct rte_vhost_vring *vring)
 933 {
 934         struct virtio_net *dev;
 935         struct vhost_virtqueue *vq;
 936
 937         dev = get_device(vid);
 938         if (dev == NULL || vring == NULL)
 939                 return -1;
 940
 941         if (vring_idx >= VHOST_MAX_VRING)
 942                 return -1;
 943
 944         vq = dev->virtqueue[vring_idx];
 945         if (!vq)
 946                 return -1;
 947
 948         if (vq_is_packed(dev)) {
 949                 vring->desc_packed = vq->desc_packed;
 950                 vring->driver_event = vq->driver_event;
 951                 vring->device_event = vq->device_event;
 952         } else {
 953                 vring->desc = vq->desc;
 954                 vring->avail = vq->avail;
 955                 vring->used = vq->used;
 956         }
 957         vring->log_guest_addr  = vq->log_guest_addr;
 958
 959         vring->callfd  = vq->callfd;
 960         vring->kickfd  = vq->kickfd;
 961         vring->size    = vq->size;
 962
 963         return 0;
 964 }
 965
 966 int
 967 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx,
 968                                   struct rte_vhost_ring_inflight *vring)
 969 {
 970         struct virtio_net *dev;
 971         struct vhost_virtqueue *vq;
 972
 973         dev = get_device(vid);
 974         if (unlikely(!dev))
 975                 return -1;
 976
 977         if (vring_idx >= VHOST_MAX_VRING)
 978                 return -1;
 979
 980         vq = dev->virtqueue[vring_idx];
 981         if (unlikely(!vq))
 982                 return -1;
 983
 984         if (vq_is_packed(dev)) {
 985                 if (unlikely(!vq->inflight_packed))
 986                         return -1;
 987
 988                 vring->inflight_packed = vq->inflight_packed;
 989         } else {
 990                 if (unlikely(!vq->inflight_split))
 991                         return -1;
 992
 993                 vring->inflight_split = vq->inflight_split;
 994         }
 995
 996         vring->resubmit_inflight = vq->resubmit_inflight;
 997
 998         return 0;
 999 }
1000
1001 int
1002 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx,
1003                                   uint16_t idx)
1004 {
1005         struct vhost_virtqueue *vq;
1006         struct virtio_net *dev;
1007
1008         dev = get_device(vid);
1009         if (unlikely(!dev))
1010                 return -1;
1011
1012         if (unlikely(!(dev->protocol_features &
1013             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1014                 return 0;
1015
1016         if (unlikely(vq_is_packed(dev)))
1017                 return -1;
1018
1019         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1020                 return -1;
1021
1022         vq = dev->virtqueue[vring_idx];
1023         if (unlikely(!vq))
1024                 return -1;
1025
1026         if (unlikely(!vq->inflight_split))
1027                 return -1;
1028
1029         if (unlikely(idx >= vq->size))
1030                 return -1;
1031
1032         vq->inflight_split->desc[idx].counter = vq->global_counter++;
1033         vq->inflight_split->desc[idx].inflight = 1;
1034         return 0;
1035 }
1036
1037 int
1038 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx,
1039                                    uint16_t head, uint16_t last,
1040                                    uint16_t *inflight_entry)
1041 {
1042         struct rte_vhost_inflight_info_packed *inflight_info;
1043         struct virtio_net *dev;
1044         struct vhost_virtqueue *vq;
1045         struct vring_packed_desc *desc;
1046         uint16_t old_free_head, free_head;
1047
1048         dev = get_device(vid);
1049         if (unlikely(!dev))
1050                 return -1;
1051
1052         if (unlikely(!(dev->protocol_features &
1053             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1054                 return 0;
1055
1056         if (unlikely(!vq_is_packed(dev)))
1057                 return -1;
1058
1059         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1060                 return -1;
1061
1062         vq = dev->virtqueue[vring_idx];
1063         if (unlikely(!vq))
1064                 return -1;
1065
1066         inflight_info = vq->inflight_packed;
1067         if (unlikely(!inflight_info))
1068                 return -1;
1069
1070         if (unlikely(head >= vq->size))
1071                 return -1;
1072
1073         desc = vq->desc_packed;
1074         old_free_head = inflight_info->old_free_head;
1075         if (unlikely(old_free_head >= vq->size))
1076                 return -1;
1077
1078         free_head = old_free_head;
1079
1080         /* init header descriptor */
1081         inflight_info->desc[old_free_head].num = 0;
1082         inflight_info->desc[old_free_head].counter = vq->global_counter++;
1083         inflight_info->desc[old_free_head].inflight = 1;
1084
1085         /* save desc entry in flight entry */
1086         while (head != ((last + 1) % vq->size)) {
1087                 inflight_info->desc[old_free_head].num++;
1088                 inflight_info->desc[free_head].addr = desc[head].addr;
1089                 inflight_info->desc[free_head].len = desc[head].len;
1090                 inflight_info->desc[free_head].flags = desc[head].flags;
1091                 inflight_info->desc[free_head].id = desc[head].id;
1092
1093                 inflight_info->desc[old_free_head].last = free_head;
1094                 free_head = inflight_info->desc[free_head].next;
1095                 inflight_info->free_head = free_head;
1096                 head = (head + 1) % vq->size;
1097         }
1098
1099         inflight_info->old_free_head = free_head;
1100         *inflight_entry = old_free_head;
1101
1102         return 0;
1103 }
1104
1105 int
1106 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
1107                                   uint16_t last_used_idx, uint16_t idx)
1108 {
1109         struct virtio_net *dev;
1110         struct vhost_virtqueue *vq;
1111
1112         dev = get_device(vid);
1113         if (unlikely(!dev))
1114                 return -1;
1115
1116         if (unlikely(!(dev->protocol_features &
1117             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1118                 return 0;
1119
1120         if (unlikely(vq_is_packed(dev)))
1121                 return -1;
1122
1123         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1124                 return -1;
1125
1126         vq = dev->virtqueue[vring_idx];
1127         if (unlikely(!vq))
1128                 return -1;
1129
1130         if (unlikely(!vq->inflight_split))
1131                 return -1;
1132
1133         if (unlikely(idx >= vq->size))
1134                 return -1;
1135
1136         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1137
1138         vq->inflight_split->desc[idx].inflight = 0;
1139
1140         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1141
1142         vq->inflight_split->used_idx = last_used_idx;
1143         return 0;
1144 }
1145
1146 int
1147 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx,
1148                                    uint16_t head)
1149 {
1150         struct rte_vhost_inflight_info_packed *inflight_info;
1151         struct virtio_net *dev;
1152         struct vhost_virtqueue *vq;
1153
1154         dev = get_device(vid);
1155         if (unlikely(!dev))
1156                 return -1;
1157
1158         if (unlikely(!(dev->protocol_features &
1159             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1160                 return 0;
1161
1162         if (unlikely(!vq_is_packed(dev)))
1163                 return -1;
1164
1165         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1166                 return -1;
1167
1168         vq = dev->virtqueue[vring_idx];
1169         if (unlikely(!vq))
1170                 return -1;
1171
1172         inflight_info = vq->inflight_packed;
1173         if (unlikely(!inflight_info))
1174                 return -1;
1175
1176         if (unlikely(head >= vq->size))
1177                 return -1;
1178
1179         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1180
1181         inflight_info->desc[head].inflight = 0;
1182
1183         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1184
1185         inflight_info->old_free_head = inflight_info->free_head;
1186         inflight_info->old_used_idx = inflight_info->used_idx;
1187         inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter;
1188
1189         return 0;
1190 }
1191
1192 int
1193 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx,
1194                                      uint16_t idx)
1195 {
1196         struct virtio_net *dev;
1197         struct vhost_virtqueue *vq;
1198
1199         dev = get_device(vid);
1200         if (unlikely(!dev))
1201                 return -1;
1202
1203         if (unlikely(!(dev->protocol_features &
1204             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1205                 return 0;
1206
1207         if (unlikely(vq_is_packed(dev)))
1208                 return -1;
1209
1210         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1211                 return -1;
1212
1213         vq = dev->virtqueue[vring_idx];
1214         if (unlikely(!vq))
1215                 return -1;
1216
1217         if (unlikely(!vq->inflight_split))
1218                 return -1;
1219
1220         if (unlikely(idx >= vq->size))
1221                 return -1;
1222
1223         vq->inflight_split->last_inflight_io = idx;
1224         return 0;
1225 }
1226
1227 int
1228 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx,
1229                                       uint16_t head)
1230 {
1231         struct rte_vhost_inflight_info_packed *inflight_info;
1232         struct virtio_net *dev;
1233         struct vhost_virtqueue *vq;
1234         uint16_t last;
1235
1236         dev = get_device(vid);
1237         if (unlikely(!dev))
1238                 return -1;
1239
1240         if (unlikely(!(dev->protocol_features &
1241             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1242                 return 0;
1243
1244         if (unlikely(!vq_is_packed(dev)))
1245                 return -1;
1246
1247         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1248                 return -1;
1249
1250         vq = dev->virtqueue[vring_idx];
1251         if (unlikely(!vq))
1252                 return -1;
1253
1254         inflight_info = vq->inflight_packed;
1255         if (unlikely(!inflight_info))
1256                 return -1;
1257
1258         if (unlikely(head >= vq->size))
1259                 return -1;
1260
1261         last = inflight_info->desc[head].last;
1262         if (unlikely(last >= vq->size))
1263                 return -1;
1264
1265         inflight_info->desc[last].next = inflight_info->free_head;
1266         inflight_info->free_head = head;
1267         inflight_info->used_idx += inflight_info->desc[head].num;
1268         if (inflight_info->used_idx >= inflight_info->desc_num) {
1269                 inflight_info->used_idx -= inflight_info->desc_num;
1270                 inflight_info->used_wrap_counter =
1271                         !inflight_info->used_wrap_counter;
1272         }
1273
1274         return 0;
1275 }
1276
1277 int
1278 rte_vhost_vring_call(int vid, uint16_t vring_idx)
1279 {
1280         struct virtio_net *dev;
1281         struct vhost_virtqueue *vq;
1282
1283         dev = get_device(vid);
1284         if (!dev)
1285                 return -1;
1286
1287         if (vring_idx >= VHOST_MAX_VRING)
1288                 return -1;
1289
1290         vq = dev->virtqueue[vring_idx];
1291         if (!vq)
1292                 return -1;
1293
1294         rte_spinlock_lock(&vq->access_lock);
1295
1296         if (vq_is_packed(dev))
1297                 vhost_vring_call_packed(dev, vq);
1298         else
1299                 vhost_vring_call_split(dev, vq);
1300
1301         rte_spinlock_unlock(&vq->access_lock);
1302
1303         return 0;
1304 }
1305
1306 uint16_t
1307 rte_vhost_avail_entries(int vid, uint16_t queue_id)
1308 {
1309         struct virtio_net *dev;
1310         struct vhost_virtqueue *vq;
1311         uint16_t ret = 0;
1312
1313         dev = get_device(vid);
1314         if (!dev)
1315                 return 0;
1316
1317         if (queue_id >= VHOST_MAX_VRING)
1318                 return 0;
1319
1320         vq = dev->virtqueue[queue_id];
1321         if (!vq)
1322                 return 0;
1323
1324         rte_spinlock_lock(&vq->access_lock);
1325
1326         if (unlikely(!vq->enabled || vq->avail == NULL))
1327                 goto out;
1328
1329         ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx;
1330
1331 out:
1332         rte_spinlock_unlock(&vq->access_lock);
1333         return ret;
1334 }
1335
1336 static inline int
1337 vhost_enable_notify_split(struct virtio_net *dev,
1338                 struct vhost_virtqueue *vq, int enable)
1339 {
1340         if (vq->used == NULL)
1341                 return -1;
1342
1343         if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
1344                 if (enable)
1345                         vq->used->flags &= ~VRING_USED_F_NO_NOTIFY;
1346                 else
1347                         vq->used->flags |= VRING_USED_F_NO_NOTIFY;
1348         } else {
1349                 if (enable)
1350                         vhost_avail_event(vq) = vq->last_avail_idx;
1351         }
1352         return 0;
1353 }
1354
1355 static inline int
1356 vhost_enable_notify_packed(struct virtio_net *dev,
1357                 struct vhost_virtqueue *vq, int enable)
1358 {
1359         uint16_t flags;
1360
1361         if (vq->device_event == NULL)
1362                 return -1;
1363
1364         if (!enable) {
1365                 vq->device_event->flags = VRING_EVENT_F_DISABLE;
1366                 return 0;
1367         }
1368
1369         flags = VRING_EVENT_F_ENABLE;
1370         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
1371                 flags = VRING_EVENT_F_DESC;
1372                 vq->device_event->off_wrap = vq->last_avail_idx |
1373                         vq->avail_wrap_counter << 15;
1374         }
1375
1376         rte_atomic_thread_fence(__ATOMIC_RELEASE);
1377
1378         vq->device_event->flags = flags;
1379         return 0;
1380 }
1381
1382 int
1383 vhost_enable_guest_notification(struct virtio_net *dev,
1384                 struct vhost_virtqueue *vq, int enable)
1385 {
1386         /*
1387          * If the virtqueue is not ready yet, it will be applied
1388          * when it will become ready.
1389          */
1390         if (!vq->ready)
1391                 return 0;
1392
1393         if (vq_is_packed(dev))
1394                 return vhost_enable_notify_packed(dev, vq, enable);
1395         else
1396                 return vhost_enable_notify_split(dev, vq, enable);
1397 }
1398
1399 int
1400 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
1401 {
1402         struct virtio_net *dev = get_device(vid);
1403         struct vhost_virtqueue *vq;
1404         int ret;
1405
1406         if (!dev)
1407                 return -1;
1408
1409         if (queue_id >= VHOST_MAX_VRING)
1410                 return -1;
1411
1412         vq = dev->virtqueue[queue_id];
1413         if (!vq)
1414                 return -1;
1415
1416         rte_spinlock_lock(&vq->access_lock);
1417
1418         vq->notif_enable = enable;
1419         ret = vhost_enable_guest_notification(dev, vq, enable);
1420
1421         rte_spinlock_unlock(&vq->access_lock);
1422
1423         return ret;
1424 }
1425
1426 void
1427 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len)
1428 {
1429         struct virtio_net *dev = get_device(vid);
1430
1431         if (dev == NULL)
1432                 return;
1433
1434         vhost_log_write(dev, addr, len);
1435 }
1436
1437 void
1438 rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
1439                          uint64_t offset, uint64_t len)
1440 {
1441         struct virtio_net *dev;
1442         struct vhost_virtqueue *vq;
1443
1444         dev = get_device(vid);
1445         if (dev == NULL)
1446                 return;
1447
1448         if (vring_idx >= VHOST_MAX_VRING)
1449                 return;
1450         vq = dev->virtqueue[vring_idx];
1451         if (!vq)
1452                 return;
1453
1454         vhost_log_used_vring(dev, vq, offset, len);
1455 }
1456
1457 uint32_t
1458 rte_vhost_rx_queue_count(int vid, uint16_t qid)
1459 {
1460         struct virtio_net *dev;
1461         struct vhost_virtqueue *vq;
1462         uint32_t ret = 0;
1463
1464         dev = get_device(vid);
1465         if (dev == NULL)
1466                 return 0;
1467
1468         if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) {
1469                 VHOST_LOG_DATA(ERR, "(%s) %s: invalid virtqueue idx %d.\n",
1470                         dev->ifname, __func__, qid);
1471                 return 0;
1472         }
1473
1474         vq = dev->virtqueue[qid];
1475         if (vq == NULL)
1476                 return 0;
1477
1478         rte_spinlock_lock(&vq->access_lock);
1479
1480         if (unlikely(!vq->enabled || vq->avail == NULL))
1481                 goto out;
1482
1483         ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx;
1484
1485 out:
1486         rte_spinlock_unlock(&vq->access_lock);
1487         return ret;
1488 }
1489
1490 struct rte_vdpa_device *
1491 rte_vhost_get_vdpa_device(int vid)
1492 {
1493         struct virtio_net *dev = get_device(vid);
1494
1495         if (dev == NULL)
1496                 return NULL;
1497
1498         return dev->vdpa_dev;
1499 }
1500
1501 int
1502 rte_vhost_get_log_base(int vid, uint64_t *log_base,
1503                 uint64_t *log_size)
1504 {
1505         struct virtio_net *dev = get_device(vid);
1506
1507         if (dev == NULL || log_base == NULL || log_size == NULL)
1508                 return -1;
1509
1510         *log_base = dev->log_base;
1511         *log_size = dev->log_size;
1512
1513         return 0;
1514 }
1515
1516 int
1517 rte_vhost_get_vring_base(int vid, uint16_t queue_id,
1518                 uint16_t *last_avail_idx, uint16_t *last_used_idx)
1519 {
1520         struct vhost_virtqueue *vq;
1521         struct virtio_net *dev = get_device(vid);
1522
1523         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1524                 return -1;
1525
1526         if (queue_id >= VHOST_MAX_VRING)
1527                 return -1;
1528
1529         vq = dev->virtqueue[queue_id];
1530         if (!vq)
1531                 return -1;
1532
1533         if (vq_is_packed(dev)) {
1534                 *last_avail_idx = (vq->avail_wrap_counter << 15) |
1535                                   vq->last_avail_idx;
1536                 *last_used_idx = (vq->used_wrap_counter << 15) |
1537                                  vq->last_used_idx;
1538         } else {
1539                 *last_avail_idx = vq->last_avail_idx;
1540                 *last_used_idx = vq->last_used_idx;
1541         }
1542
1543         return 0;
1544 }
1545
1546 int
1547 rte_vhost_set_vring_base(int vid, uint16_t queue_id,
1548                 uint16_t last_avail_idx, uint16_t last_used_idx)
1549 {
1550         struct vhost_virtqueue *vq;
1551         struct virtio_net *dev = get_device(vid);
1552
1553         if (!dev)
1554                 return -1;
1555
1556         if (queue_id >= VHOST_MAX_VRING)
1557                 return -1;
1558
1559         vq = dev->virtqueue[queue_id];
1560         if (!vq)
1561                 return -1;
1562
1563         if (vq_is_packed(dev)) {
1564                 vq->last_avail_idx = last_avail_idx & 0x7fff;
1565                 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15));
1566                 vq->last_used_idx = last_used_idx & 0x7fff;
1567                 vq->used_wrap_counter = !!(last_used_idx & (1 << 15));
1568         } else {
1569                 vq->last_avail_idx = last_avail_idx;
1570                 vq->last_used_idx = last_used_idx;
1571         }
1572
1573         return 0;
1574 }
1575
1576 int
1577 rte_vhost_get_vring_base_from_inflight(int vid,
1578                                        uint16_t queue_id,
1579                                        uint16_t *last_avail_idx,
1580                                        uint16_t *last_used_idx)
1581 {
1582         struct rte_vhost_inflight_info_packed *inflight_info;
1583         struct vhost_virtqueue *vq;
1584         struct virtio_net *dev = get_device(vid);
1585
1586         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1587                 return -1;
1588
1589         if (queue_id >= VHOST_MAX_VRING)
1590                 return -1;
1591
1592         vq = dev->virtqueue[queue_id];
1593         if (!vq)
1594                 return -1;
1595
1596         if (!vq_is_packed(dev))
1597                 return -1;
1598
1599         inflight_info = vq->inflight_packed;
1600         if (!inflight_info)
1601                 return -1;
1602
1603         *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) |
1604                           inflight_info->old_used_idx;
1605         *last_used_idx = *last_avail_idx;
1606
1607         return 0;
1608 }
1609
1610 int
1611 rte_vhost_extern_callback_register(int vid,
1612                 struct rte_vhost_user_extern_ops const * const ops, void *ctx)
1613 {
1614         struct virtio_net *dev = get_device(vid);
1615
1616         if (dev == NULL || ops == NULL)
1617                 return -1;
1618
1619         dev->extern_ops = *ops;
1620         dev->extern_data = ctx;
1621         return 0;
1622 }
1623
1624 static __rte_always_inline int
1625 async_channel_register(int vid, uint16_t queue_id)
1626 {
1627         struct virtio_net *dev = get_device(vid);
1628         struct vhost_virtqueue *vq = dev->virtqueue[queue_id];
1629         struct vhost_async *async;
1630         int node = vq->numa_node;
1631
1632         if (unlikely(vq->async)) {
1633                 VHOST_LOG_CONFIG(ERR,
1634                                 "(%s) async register failed: already registered (qid: %d)\n",
1635                                 dev->ifname, queue_id);
1636                 return -1;
1637         }
1638
1639         async = rte_zmalloc_socket(NULL, sizeof(struct vhost_async), 0, node);
1640         if (!async) {
1641                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async metadata (qid: %d)\n",
1642                                 dev->ifname, queue_id);
1643                 return -1;
1644         }
1645
1646         async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info),
1647                         RTE_CACHE_LINE_SIZE, node);
1648         if (!async->pkts_info) {
1649                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async_pkts_info (qid: %d)\n",
1650                                 dev->ifname, queue_id);
1651                 goto out_free_async;
1652         }
1653
1654         async->pkts_cmpl_flag = rte_zmalloc_socket(NULL, vq->size * sizeof(bool),
1655                         RTE_CACHE_LINE_SIZE, node);
1656         if (!async->pkts_cmpl_flag) {
1657                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async pkts_cmpl_flag (qid: %d)\n",
1658                                 dev->ifname, queue_id);
1659                 goto out_free_async;
1660         }
1661
1662         if (vq_is_packed(dev)) {
1663                 async->buffers_packed = rte_malloc_socket(NULL,
1664                                 vq->size * sizeof(struct vring_used_elem_packed),
1665                                 RTE_CACHE_LINE_SIZE, node);
1666                 if (!async->buffers_packed) {
1667                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async buffers (qid: %d)\n",
1668                                         dev->ifname, queue_id);
1669                         goto out_free_inflight;
1670                 }
1671         } else {
1672                 async->descs_split = rte_malloc_socket(NULL,
1673                                 vq->size * sizeof(struct vring_used_elem),
1674                                 RTE_CACHE_LINE_SIZE, node);
1675                 if (!async->descs_split) {
1676                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async descs (qid: %d)\n",
1677                                         dev->ifname, queue_id);
1678                         goto out_free_inflight;
1679                 }
1680         }
1681
1682         vq->async = async;
1683
1684         return 0;
1685 out_free_inflight:
1686         rte_free(async->pkts_info);
1687 out_free_async:
1688         rte_free(async);
1689
1690         return -1;
1691 }
1692
1693 int
1694 rte_vhost_async_channel_register(int vid, uint16_t queue_id)
1695 {
1696         struct vhost_virtqueue *vq;
1697         struct virtio_net *dev = get_device(vid);
1698         int ret;
1699
1700         if (dev == NULL)
1701                 return -1;
1702
1703         if (queue_id >= VHOST_MAX_VRING)
1704                 return -1;
1705
1706         vq = dev->virtqueue[queue_id];
1707
1708         if (unlikely(vq == NULL || !dev->async_copy))
1709                 return -1;
1710
1711         rte_spinlock_lock(&vq->access_lock);
1712         ret = async_channel_register(vid, queue_id);
1713         rte_spinlock_unlock(&vq->access_lock);
1714
1715         return ret;
1716 }
1717
1718 int
1719 rte_vhost_async_channel_register_thread_unsafe(int vid, uint16_t queue_id)
1720 {
1721         struct vhost_virtqueue *vq;
1722         struct virtio_net *dev = get_device(vid);
1723
1724         if (dev == NULL)
1725                 return -1;
1726
1727         if (queue_id >= VHOST_MAX_VRING)
1728                 return -1;
1729
1730         vq = dev->virtqueue[queue_id];
1731
1732         if (unlikely(vq == NULL || !dev->async_copy))
1733                 return -1;
1734
1735         return async_channel_register(vid, queue_id);
1736 }
1737
1738 int
1739 rte_vhost_async_channel_unregister(int vid, uint16_t queue_id)
1740 {
1741         struct vhost_virtqueue *vq;
1742         struct virtio_net *dev = get_device(vid);
1743         int ret = -1;
1744
1745         if (dev == NULL)
1746                 return ret;
1747
1748         if (queue_id >= VHOST_MAX_VRING)
1749                 return ret;
1750
1751         vq = dev->virtqueue[queue_id];
1752
1753         if (vq == NULL)
1754                 return ret;
1755
1756         ret = 0;
1757
1758         if (!vq->async)
1759                 return ret;
1760
1761         if (!rte_spinlock_trylock(&vq->access_lock)) {
1762                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel, virtqueue busy.\n",
1763                                 dev->ifname);
1764                 return -1;
1765         }
1766
1767         if (vq->async->pkts_inflight_n) {
1768                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname);
1769                 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n",
1770                         dev->ifname);
1771                 ret = -1;
1772                 goto out;
1773         }
1774
1775         vhost_free_async_mem(vq);
1776 out:
1777         rte_spinlock_unlock(&vq->access_lock);
1778
1779         return ret;
1780 }
1781
1782 int
1783 rte_vhost_async_channel_unregister_thread_unsafe(int vid, uint16_t queue_id)
1784 {
1785         struct vhost_virtqueue *vq;
1786         struct virtio_net *dev = get_device(vid);
1787
1788         if (dev == NULL)
1789                 return -1;
1790
1791         if (queue_id >= VHOST_MAX_VRING)
1792                 return -1;
1793
1794         vq = dev->virtqueue[queue_id];
1795
1796         if (vq == NULL)
1797                 return -1;
1798
1799         if (!vq->async)
1800                 return 0;
1801
1802         if (vq->async->pkts_inflight_n) {
1803                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname);
1804                 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n",
1805                         dev->ifname);
1806                 return -1;
1807         }
1808
1809         vhost_free_async_mem(vq);
1810
1811         return 0;
1812 }
1813
1814 int
1815 rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id)
1816 {
1817         struct rte_dma_info info;
1818         void *pkts_cmpl_flag_addr;
1819         uint16_t max_desc;
1820
1821         if (!rte_dma_is_valid(dma_id)) {
1822                 VHOST_LOG_CONFIG(ERR, "DMA %d is not found.\n", dma_id);
1823                 return -1;
1824         }
1825
1826         rte_dma_info_get(dma_id, &info);
1827         if (vchan_id >= info.max_vchans) {
1828                 VHOST_LOG_CONFIG(ERR, "Invalid DMA %d vChannel %u.\n", dma_id, vchan_id);
1829                 return -1;
1830         }
1831
1832         if (!dma_copy_track[dma_id].vchans) {
1833                 struct async_dma_vchan_info *vchans;
1834
1835                 vchans = rte_zmalloc(NULL, sizeof(struct async_dma_vchan_info) * info.max_vchans,
1836                                 RTE_CACHE_LINE_SIZE);
1837                 if (vchans == NULL) {
1838                         VHOST_LOG_CONFIG(ERR, "Failed to allocate vchans for DMA %d vChannel %u.\n",
1839                                         dma_id, vchan_id);
1840                         return -1;
1841                 }
1842
1843                 dma_copy_track[dma_id].vchans = vchans;
1844         }
1845
1846         if (dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) {
1847                 VHOST_LOG_CONFIG(INFO, "DMA %d vChannel %u already registered.\n", dma_id,
1848                                 vchan_id);
1849                 return 0;
1850         }
1851
1852         max_desc = info.max_desc;
1853         if (!rte_is_power_of_2(max_desc))
1854                 max_desc = rte_align32pow2(max_desc);
1855
1856         pkts_cmpl_flag_addr = rte_zmalloc(NULL, sizeof(bool *) * max_desc, RTE_CACHE_LINE_SIZE);
1857         if (!pkts_cmpl_flag_addr) {
1858                 VHOST_LOG_CONFIG(ERR, "Failed to allocate pkts_cmpl_flag_addr for DMA %d "
1859                                 "vChannel %u.\n", dma_id, vchan_id);
1860
1861                 if (dma_copy_track[dma_id].nr_vchans == 0) {
1862                         rte_free(dma_copy_track[dma_id].vchans);
1863                         dma_copy_track[dma_id].vchans = NULL;
1864                 }
1865                 return -1;
1866         }
1867
1868         dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = pkts_cmpl_flag_addr;
1869         dma_copy_track[dma_id].vchans[vchan_id].ring_size = max_desc;
1870         dma_copy_track[dma_id].vchans[vchan_id].ring_mask = max_desc - 1;
1871         dma_copy_track[dma_id].nr_vchans++;
1872
1873         return 0;
1874 }
1875
1876 int
1877 rte_vhost_async_get_inflight(int vid, uint16_t queue_id)
1878 {
1879         struct vhost_virtqueue *vq;
1880         struct virtio_net *dev = get_device(vid);
1881         int ret = -1;
1882
1883         if (dev == NULL)
1884                 return ret;
1885
1886         if (queue_id >= VHOST_MAX_VRING)
1887                 return ret;
1888
1889         vq = dev->virtqueue[queue_id];
1890
1891         if (vq == NULL)
1892                 return ret;
1893
1894         if (!vq->async)
1895                 return ret;
1896
1897         if (!rte_spinlock_trylock(&vq->access_lock)) {
1898                 VHOST_LOG_CONFIG(DEBUG,
1899                         "(%s) failed to check in-flight packets. virtqueue busy.\n",
1900                         dev->ifname);
1901                 return ret;
1902         }
1903
1904         ret = vq->async->pkts_inflight_n;
1905         rte_spinlock_unlock(&vq->access_lock);
1906
1907         return ret;
1908 }
1909
1910 int
1911 rte_vhost_async_get_inflight_thread_unsafe(int vid, uint16_t queue_id)
1912 {
1913         struct vhost_virtqueue *vq;
1914         struct virtio_net *dev = get_device(vid);
1915         int ret = -1;
1916
1917         if (dev == NULL)
1918                 return ret;
1919
1920         if (queue_id >= VHOST_MAX_VRING)
1921                 return ret;
1922
1923         vq = dev->virtqueue[queue_id];
1924
1925         if (vq == NULL)
1926                 return ret;
1927
1928         if (!vq->async)
1929                 return ret;
1930
1931         ret = vq->async->pkts_inflight_n;
1932
1933         return ret;
1934 }
1935
1936 int
1937 rte_vhost_get_monitor_addr(int vid, uint16_t queue_id,
1938                 struct rte_vhost_power_monitor_cond *pmc)
1939 {
1940         struct virtio_net *dev = get_device(vid);
1941         struct vhost_virtqueue *vq;
1942
1943         if (dev == NULL)
1944                 return -1;
1945         if (queue_id >= VHOST_MAX_VRING)
1946                 return -1;
1947
1948         vq = dev->virtqueue[queue_id];
1949         if (vq == NULL)
1950                 return -1;
1951
1952         if (vq_is_packed(dev)) {
1953                 struct vring_packed_desc *desc;
1954                 desc = vq->desc_packed;
1955                 pmc->addr = &desc[vq->last_avail_idx].flags;
1956                 if (vq->avail_wrap_counter)
1957                         pmc->val = VRING_DESC_F_AVAIL;
1958                 else
1959                         pmc->val = VRING_DESC_F_USED;
1960                 pmc->mask = VRING_DESC_F_AVAIL | VRING_DESC_F_USED;
1961                 pmc->size = sizeof(desc[vq->last_avail_idx].flags);
1962                 pmc->match = 1;
1963         } else {
1964                 pmc->addr = &vq->avail->idx;
1965                 pmc->val = vq->last_avail_idx & (vq->size - 1);
1966                 pmc->mask = vq->size - 1;
1967                 pmc->size = sizeof(vq->avail->idx);
1968                 pmc->match = 0;
1969         }
1970
1971         return 0;
1972 }
1973
1974 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO);
1975 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING);