lib/vhost/vhost.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <linux/vhost.h>
   6 #include <linux/virtio_net.h>
   7 #include <stdint.h>
   8 #include <stdlib.h>
   9 #ifdef RTE_LIBRTE_VHOST_NUMA
  10 #include <numa.h>
  11 #include <numaif.h>
  12 #endif
  13
  14 #include <rte_errno.h>
  15 #include <rte_log.h>
  16 #include <rte_memory.h>
  17 #include <rte_malloc.h>
  18 #include <rte_vhost.h>
  19
  20 #include "iotlb.h"
  21 #include "vhost.h"
  22 #include "vhost_user.h"
  23
  24 struct virtio_net *vhost_devices[RTE_MAX_VHOST_DEVICE];
  25 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER;
  26
  27 struct vhost_vq_stats_name_off {
  28         char name[RTE_VHOST_STATS_NAME_SIZE];
  29         unsigned int offset;
  30 };
  31
  32 static const struct vhost_vq_stats_name_off vhost_vq_stat_strings[] = {
  33         {"good_packets",           offsetof(struct vhost_virtqueue, stats.packets)},
  34         {"good_bytes",             offsetof(struct vhost_virtqueue, stats.bytes)},
  35         {"multicast_packets",      offsetof(struct vhost_virtqueue, stats.multicast)},
  36         {"broadcast_packets",      offsetof(struct vhost_virtqueue, stats.broadcast)},
  37         {"undersize_packets",      offsetof(struct vhost_virtqueue, stats.size_bins[0])},
  38         {"size_64_packets",        offsetof(struct vhost_virtqueue, stats.size_bins[1])},
  39         {"size_65_127_packets",    offsetof(struct vhost_virtqueue, stats.size_bins[2])},
  40         {"size_128_255_packets",   offsetof(struct vhost_virtqueue, stats.size_bins[3])},
  41         {"size_256_511_packets",   offsetof(struct vhost_virtqueue, stats.size_bins[4])},
  42         {"size_512_1023_packets",  offsetof(struct vhost_virtqueue, stats.size_bins[5])},
  43         {"size_1024_1518_packets", offsetof(struct vhost_virtqueue, stats.size_bins[6])},
  44         {"size_1519_max_packets",  offsetof(struct vhost_virtqueue, stats.size_bins[7])},
  45         {"guest_notifications",    offsetof(struct vhost_virtqueue, stats.guest_notifications)},
  46         {"iotlb_hits",             offsetof(struct vhost_virtqueue, stats.iotlb_hits)},
  47         {"iotlb_misses",           offsetof(struct vhost_virtqueue, stats.iotlb_misses)},
  48         {"inflight_submitted",     offsetof(struct vhost_virtqueue, stats.inflight_submitted)},
  49         {"inflight_completed",     offsetof(struct vhost_virtqueue, stats.inflight_completed)},
  50 };
  51
  52 #define VHOST_NB_VQ_STATS RTE_DIM(vhost_vq_stat_strings)
  53
  54 /* Called with iotlb_lock read-locked */
  55 uint64_t
  56 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
  57                     uint64_t iova, uint64_t *size, uint8_t perm)
  58 {
  59         uint64_t vva, tmp_size;
  60
  61         if (unlikely(!*size))
  62                 return 0;
  63
  64         tmp_size = *size;
  65
  66         vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm);
  67         if (tmp_size == *size) {
  68                 if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
  69                         vq->stats.iotlb_hits++;
  70                 return vva;
  71         }
  72
  73         if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
  74                 vq->stats.iotlb_misses++;
  75
  76         iova += tmp_size;
  77
  78         if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) {
  79                 /*
  80                  * iotlb_lock is read-locked for a full burst,
  81                  * but it only protects the iotlb cache.
  82                  * In case of IOTLB miss, we might block on the socket,
  83                  * which could cause a deadlock with QEMU if an IOTLB update
  84                  * is being handled. We can safely unlock here to avoid it.
  85                  */
  86                 vhost_user_iotlb_rd_unlock(vq);
  87
  88                 vhost_user_iotlb_pending_insert(dev, vq, iova, perm);
  89                 if (vhost_user_iotlb_miss(dev, iova, perm)) {
  90                         VHOST_LOG_DATA(ERR, "(%s) IOTLB miss req failed for IOVA 0x%" PRIx64 "\n",
  91                                 dev->ifname, iova);
  92                         vhost_user_iotlb_pending_remove(vq, iova, 1, perm);
  93                 }
  94
  95                 vhost_user_iotlb_rd_lock(vq);
  96         }
  97
  98         return 0;
  99 }
 100
 101 #define VHOST_LOG_PAGE  4096
 102
 103 /*
 104  * Atomically set a bit in memory.
 105  */
 106 static __rte_always_inline void
 107 vhost_set_bit(unsigned int nr, volatile uint8_t *addr)
 108 {
 109 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
 110         /*
 111          * __sync_ built-ins are deprecated, but __atomic_ ones
 112          * are sub-optimized in older GCC versions.
 113          */
 114         __sync_fetch_and_or_1(addr, (1U << nr));
 115 #else
 116         __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED);
 117 #endif
 118 }
 119
 120 static __rte_always_inline void
 121 vhost_log_page(uint8_t *log_base, uint64_t page)
 122 {
 123         vhost_set_bit(page % 8, &log_base[page / 8]);
 124 }
 125
 126 void
 127 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
 128 {
 129         uint64_t page;
 130
 131         if (unlikely(!dev->log_base || !len))
 132                 return;
 133
 134         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 135                 return;
 136
 137         /* To make sure guest memory updates are committed before logging */
 138         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 139
 140         page = addr / VHOST_LOG_PAGE;
 141         while (page * VHOST_LOG_PAGE < addr + len) {
 142                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 143                 page += 1;
 144         }
 145 }
 146
 147 void
 148 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 149                              uint64_t iova, uint64_t len)
 150 {
 151         uint64_t hva, gpa, map_len;
 152         map_len = len;
 153
 154         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 155         if (map_len != len) {
 156                 VHOST_LOG_DATA(ERR,
 157                         "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 158                         dev->ifname, iova);
 159                 return;
 160         }
 161
 162         gpa = hva_to_gpa(dev, hva, len);
 163         if (gpa)
 164                 __vhost_log_write(dev, gpa, len);
 165 }
 166
 167 void
 168 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq)
 169 {
 170         unsigned long *log_base;
 171         int i;
 172
 173         if (unlikely(!dev->log_base))
 174                 return;
 175
 176         /* No cache, nothing to sync */
 177         if (unlikely(!vq->log_cache))
 178                 return;
 179
 180         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 181
 182         log_base = (unsigned long *)(uintptr_t)dev->log_base;
 183
 184         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 185                 struct log_cache_entry *elem = vq->log_cache + i;
 186
 187 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100)
 188                 /*
 189                  * '__sync' builtins are deprecated, but '__atomic' ones
 190                  * are sub-optimized in older GCC versions.
 191                  */
 192                 __sync_fetch_and_or(log_base + elem->offset, elem->val);
 193 #else
 194                 __atomic_fetch_or(log_base + elem->offset, elem->val,
 195                                 __ATOMIC_RELAXED);
 196 #endif
 197         }
 198
 199         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 200
 201         vq->log_cache_nb_elem = 0;
 202 }
 203
 204 static __rte_always_inline void
 205 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
 206                         uint64_t page)
 207 {
 208         uint32_t bit_nr = page % (sizeof(unsigned long) << 3);
 209         uint32_t offset = page / (sizeof(unsigned long) << 3);
 210         int i;
 211
 212         if (unlikely(!vq->log_cache)) {
 213                 /* No logging cache allocated, write dirty log map directly */
 214                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 215                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 216
 217                 return;
 218         }
 219
 220         for (i = 0; i < vq->log_cache_nb_elem; i++) {
 221                 struct log_cache_entry *elem = vq->log_cache + i;
 222
 223                 if (elem->offset == offset) {
 224                         elem->val |= (1UL << bit_nr);
 225                         return;
 226                 }
 227         }
 228
 229         if (unlikely(i >= VHOST_LOG_CACHE_NR)) {
 230                 /*
 231                  * No more room for a new log cache entry,
 232                  * so write the dirty log map directly.
 233                  */
 234                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 235                 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
 236
 237                 return;
 238         }
 239
 240         vq->log_cache[i].offset = offset;
 241         vq->log_cache[i].val = (1UL << bit_nr);
 242         vq->log_cache_nb_elem++;
 243 }
 244
 245 void
 246 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
 247                         uint64_t addr, uint64_t len)
 248 {
 249         uint64_t page;
 250
 251         if (unlikely(!dev->log_base || !len))
 252                 return;
 253
 254         if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
 255                 return;
 256
 257         page = addr / VHOST_LOG_PAGE;
 258         while (page * VHOST_LOG_PAGE < addr + len) {
 259                 vhost_log_cache_page(dev, vq, page);
 260                 page += 1;
 261         }
 262 }
 263
 264 void
 265 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
 266                              uint64_t iova, uint64_t len)
 267 {
 268         uint64_t hva, gpa, map_len;
 269         map_len = len;
 270
 271         hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
 272         if (map_len != len) {
 273                 VHOST_LOG_DATA(ERR,
 274                         "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n",
 275                         dev->ifname, iova);
 276                 return;
 277         }
 278
 279         gpa = hva_to_gpa(dev, hva, len);
 280         if (gpa)
 281                 __vhost_log_cache_write(dev, vq, gpa, len);
 282 }
 283
 284 void *
 285 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
 286                 uint64_t desc_addr, uint64_t desc_len)
 287 {
 288         void *idesc;
 289         uint64_t src, dst;
 290         uint64_t len, remain = desc_len;
 291
 292         idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node);
 293         if (unlikely(!idesc))
 294                 return NULL;
 295
 296         dst = (uint64_t)(uintptr_t)idesc;
 297
 298         while (remain) {
 299                 len = remain;
 300                 src = vhost_iova_to_vva(dev, vq, desc_addr, &len,
 301                                 VHOST_ACCESS_RO);
 302                 if (unlikely(!src || !len)) {
 303                         rte_free(idesc);
 304                         return NULL;
 305                 }
 306
 307                 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len);
 308
 309                 remain -= len;
 310                 dst += len;
 311                 desc_addr += len;
 312         }
 313
 314         return idesc;
 315 }
 316
 317 void
 318 cleanup_vq(struct vhost_virtqueue *vq, int destroy)
 319 {
 320         if ((vq->callfd >= 0) && (destroy != 0))
 321                 close(vq->callfd);
 322         if (vq->kickfd >= 0)
 323                 close(vq->kickfd);
 324 }
 325
 326 void
 327 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq)
 328 {
 329         if (!(dev->protocol_features &
 330             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
 331                 return;
 332
 333         if (vq_is_packed(dev)) {
 334                 if (vq->inflight_packed)
 335                         vq->inflight_packed = NULL;
 336         } else {
 337                 if (vq->inflight_split)
 338                         vq->inflight_split = NULL;
 339         }
 340
 341         if (vq->resubmit_inflight) {
 342                 if (vq->resubmit_inflight->resubmit_list) {
 343                         rte_free(vq->resubmit_inflight->resubmit_list);
 344                         vq->resubmit_inflight->resubmit_list = NULL;
 345                 }
 346                 rte_free(vq->resubmit_inflight);
 347                 vq->resubmit_inflight = NULL;
 348         }
 349 }
 350
 351 /*
 352  * Unmap any memory, close any file descriptors and
 353  * free any memory owned by a device.
 354  */
 355 void
 356 cleanup_device(struct virtio_net *dev, int destroy)
 357 {
 358         uint32_t i;
 359
 360         vhost_backend_cleanup(dev);
 361
 362         for (i = 0; i < dev->nr_vring; i++) {
 363                 cleanup_vq(dev->virtqueue[i], destroy);
 364                 cleanup_vq_inflight(dev, dev->virtqueue[i]);
 365         }
 366 }
 367
 368 static void
 369 vhost_free_async_mem(struct vhost_virtqueue *vq)
 370 {
 371         if (!vq->async)
 372                 return;
 373
 374         rte_free(vq->async->pkts_info);
 375         rte_free(vq->async->pkts_cmpl_flag);
 376
 377         rte_free(vq->async->buffers_packed);
 378         vq->async->buffers_packed = NULL;
 379         rte_free(vq->async->descs_split);
 380         vq->async->descs_split = NULL;
 381
 382         rte_free(vq->async);
 383         vq->async = NULL;
 384 }
 385
 386 void
 387 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 388 {
 389         if (vq_is_packed(dev))
 390                 rte_free(vq->shadow_used_packed);
 391         else
 392                 rte_free(vq->shadow_used_split);
 393
 394         vhost_free_async_mem(vq);
 395         rte_free(vq->batch_copy_elems);
 396         rte_mempool_free(vq->iotlb_pool);
 397         rte_free(vq->log_cache);
 398         rte_free(vq);
 399 }
 400
 401 /*
 402  * Release virtqueues and device memory.
 403  */
 404 static void
 405 free_device(struct virtio_net *dev)
 406 {
 407         uint32_t i;
 408
 409         for (i = 0; i < dev->nr_vring; i++)
 410                 free_vq(dev, dev->virtqueue[i]);
 411
 412         rte_free(dev);
 413 }
 414
 415 static __rte_always_inline int
 416 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 417 {
 418         if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG))))
 419                 return 0;
 420
 421         vq->log_guest_addr = translate_log_addr(dev, vq,
 422                                                 vq->ring_addrs.log_guest_addr);
 423         if (vq->log_guest_addr == 0)
 424                 return -1;
 425
 426         return 0;
 427 }
 428
 429 /*
 430  * Converts vring log address to GPA
 431  * If IOMMU is enabled, the log address is IOVA
 432  * If IOMMU not enabled, the log address is already GPA
 433  *
 434  * Caller should have iotlb_lock read-locked
 435  */
 436 uint64_t
 437 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 438                 uint64_t log_addr)
 439 {
 440         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 441                 const uint64_t exp_size = sizeof(uint64_t);
 442                 uint64_t hva, gpa;
 443                 uint64_t size = exp_size;
 444
 445                 hva = vhost_iova_to_vva(dev, vq, log_addr,
 446                                         &size, VHOST_ACCESS_RW);
 447
 448                 if (size != exp_size)
 449                         return 0;
 450
 451                 gpa = hva_to_gpa(dev, hva, exp_size);
 452                 if (!gpa) {
 453                         VHOST_LOG_DATA(ERR,
 454                                 "(%s) failed to find GPA for log_addr: 0x%"
 455                                 PRIx64 " hva: 0x%" PRIx64 "\n",
 456                                 dev->ifname, log_addr, hva);
 457                         return 0;
 458                 }
 459                 return gpa;
 460
 461         } else
 462                 return log_addr;
 463 }
 464
 465 /* Caller should have iotlb_lock read-locked */
 466 static int
 467 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 468 {
 469         uint64_t req_size, size;
 470
 471         req_size = sizeof(struct vring_desc) * vq->size;
 472         size = req_size;
 473         vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 474                                                 vq->ring_addrs.desc_user_addr,
 475                                                 &size, VHOST_ACCESS_RW);
 476         if (!vq->desc || size != req_size)
 477                 return -1;
 478
 479         req_size = sizeof(struct vring_avail);
 480         req_size += sizeof(uint16_t) * vq->size;
 481         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 482                 req_size += sizeof(uint16_t);
 483         size = req_size;
 484         vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 485                                                 vq->ring_addrs.avail_user_addr,
 486                                                 &size, VHOST_ACCESS_RW);
 487         if (!vq->avail || size != req_size)
 488                 return -1;
 489
 490         req_size = sizeof(struct vring_used);
 491         req_size += sizeof(struct vring_used_elem) * vq->size;
 492         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 493                 req_size += sizeof(uint16_t);
 494         size = req_size;
 495         vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq,
 496                                                 vq->ring_addrs.used_user_addr,
 497                                                 &size, VHOST_ACCESS_RW);
 498         if (!vq->used || size != req_size)
 499                 return -1;
 500
 501         return 0;
 502 }
 503
 504 /* Caller should have iotlb_lock read-locked */
 505 static int
 506 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)
 507 {
 508         uint64_t req_size, size;
 509
 510         req_size = sizeof(struct vring_packed_desc) * vq->size;
 511         size = req_size;
 512         vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 513                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr,
 514                                 &size, VHOST_ACCESS_RW);
 515         if (!vq->desc_packed || size != req_size)
 516                 return -1;
 517
 518         req_size = sizeof(struct vring_packed_desc_event);
 519         size = req_size;
 520         vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t)
 521                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr,
 522                                 &size, VHOST_ACCESS_RW);
 523         if (!vq->driver_event || size != req_size)
 524                 return -1;
 525
 526         req_size = sizeof(struct vring_packed_desc_event);
 527         size = req_size;
 528         vq->device_event = (struct vring_packed_desc_event *)(uintptr_t)
 529                 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr,
 530                                 &size, VHOST_ACCESS_RW);
 531         if (!vq->device_event || size != req_size)
 532                 return -1;
 533
 534         return 0;
 535 }
 536
 537 int
 538 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 539 {
 540
 541         if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
 542                 return -1;
 543
 544         if (vq_is_packed(dev)) {
 545                 if (vring_translate_packed(dev, vq) < 0)
 546                         return -1;
 547         } else {
 548                 if (vring_translate_split(dev, vq) < 0)
 549                         return -1;
 550         }
 551
 552         if (log_translate(dev, vq) < 0)
 553                 return -1;
 554
 555         vq->access_ok = true;
 556
 557         return 0;
 558 }
 559
 560 void
 561 vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 562 {
 563         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 564                 vhost_user_iotlb_wr_lock(vq);
 565
 566         vq->access_ok = false;
 567         vq->desc = NULL;
 568         vq->avail = NULL;
 569         vq->used = NULL;
 570         vq->log_guest_addr = 0;
 571
 572         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 573                 vhost_user_iotlb_wr_unlock(vq);
 574 }
 575
 576 static void
 577 init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 578 {
 579         struct vhost_virtqueue *vq;
 580         int numa_node = SOCKET_ID_ANY;
 581
 582         if (vring_idx >= VHOST_MAX_VRING) {
 583                 VHOST_LOG_CONFIG(ERR, "(%s) failed to init vring, out of bound (%d)\n",
 584                                 dev->ifname, vring_idx);
 585                 return;
 586         }
 587
 588         vq = dev->virtqueue[vring_idx];
 589         if (!vq) {
 590                 VHOST_LOG_CONFIG(ERR, "(%s) virtqueue not allocated (%d)\n",
 591                                 dev->ifname, vring_idx);
 592                 return;
 593         }
 594
 595         memset(vq, 0, sizeof(struct vhost_virtqueue));
 596
 597         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
 598         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
 599         vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
 600
 601 #ifdef RTE_LIBRTE_VHOST_NUMA
 602         if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) {
 603                 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n",
 604                         dev->ifname, rte_strerror(errno));
 605                 numa_node = SOCKET_ID_ANY;
 606         }
 607 #endif
 608         vq->numa_node = numa_node;
 609
 610         vhost_user_iotlb_init(dev, vring_idx);
 611 }
 612
 613 static void
 614 reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 615 {
 616         struct vhost_virtqueue *vq;
 617         int callfd;
 618
 619         if (vring_idx >= VHOST_MAX_VRING) {
 620                 VHOST_LOG_CONFIG(ERR,
 621                                 "(%s) failed to reset vring, out of bound (%d)\n",
 622                                 dev->ifname, vring_idx);
 623                 return;
 624         }
 625
 626         vq = dev->virtqueue[vring_idx];
 627         if (!vq) {
 628                 VHOST_LOG_CONFIG(ERR, "(%s) failed to reset vring, virtqueue not allocated (%d)\n",
 629                                 dev->ifname, vring_idx);
 630                 return;
 631         }
 632
 633         callfd = vq->callfd;
 634         init_vring_queue(dev, vring_idx);
 635         vq->callfd = callfd;
 636 }
 637
 638 int
 639 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 640 {
 641         struct vhost_virtqueue *vq;
 642         uint32_t i;
 643
 644         /* Also allocate holes, if any, up to requested vring index. */
 645         for (i = 0; i <= vring_idx; i++) {
 646                 if (dev->virtqueue[i])
 647                         continue;
 648
 649                 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0);
 650                 if (vq == NULL) {
 651                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate memory for vring %u.\n",
 652                                         dev->ifname, i);
 653                         return -1;
 654                 }
 655
 656                 dev->virtqueue[i] = vq;
 657                 init_vring_queue(dev, i);
 658                 rte_spinlock_init(&vq->access_lock);
 659                 vq->avail_wrap_counter = 1;
 660                 vq->used_wrap_counter = 1;
 661                 vq->signalled_used_valid = false;
 662         }
 663
 664         dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1);
 665
 666         return 0;
 667 }
 668
 669 /*
 670  * Reset some variables in device structure, while keeping few
 671  * others untouched, such as vid, ifname, nr_vring: they
 672  * should be same unless the device is removed.
 673  */
 674 void
 675 reset_device(struct virtio_net *dev)
 676 {
 677         uint32_t i;
 678
 679         dev->features = 0;
 680         dev->protocol_features = 0;
 681         dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 682
 683         for (i = 0; i < dev->nr_vring; i++)
 684                 reset_vring_queue(dev, i);
 685 }
 686
 687 /*
 688  * Invoked when there is a new vhost-user connection established (when
 689  * there is a new virtio device being attached).
 690  */
 691 int
 692 vhost_new_device(void)
 693 {
 694         struct virtio_net *dev;
 695         int i;
 696
 697         pthread_mutex_lock(&vhost_dev_lock);
 698         for (i = 0; i < RTE_MAX_VHOST_DEVICE; i++) {
 699                 if (vhost_devices[i] == NULL)
 700                         break;
 701         }
 702
 703         if (i == RTE_MAX_VHOST_DEVICE) {
 704                 VHOST_LOG_CONFIG(ERR, "failed to find a free slot for new device.\n");
 705                 pthread_mutex_unlock(&vhost_dev_lock);
 706                 return -1;
 707         }
 708
 709         dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0);
 710         if (dev == NULL) {
 711                 VHOST_LOG_CONFIG(ERR, "failed to allocate memory for new device.\n");
 712                 pthread_mutex_unlock(&vhost_dev_lock);
 713                 return -1;
 714         }
 715
 716         vhost_devices[i] = dev;
 717         pthread_mutex_unlock(&vhost_dev_lock);
 718
 719         dev->vid = i;
 720         dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 721         dev->slave_req_fd = -1;
 722         dev->postcopy_ufd = -1;
 723         rte_spinlock_init(&dev->slave_req_lock);
 724
 725         return i;
 726 }
 727
 728 void
 729 vhost_destroy_device_notify(struct virtio_net *dev)
 730 {
 731         struct rte_vdpa_device *vdpa_dev;
 732
 733         if (dev->flags & VIRTIO_DEV_RUNNING) {
 734                 vdpa_dev = dev->vdpa_dev;
 735                 if (vdpa_dev)
 736                         vdpa_dev->ops->dev_close(dev->vid);
 737                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 738                 dev->notify_ops->destroy_device(dev->vid);
 739         }
 740 }
 741
 742 /*
 743  * Invoked when there is the vhost-user connection is broken (when
 744  * the virtio device is being detached).
 745  */
 746 void
 747 vhost_destroy_device(int vid)
 748 {
 749         struct virtio_net *dev = get_device(vid);
 750
 751         if (dev == NULL)
 752                 return;
 753
 754         vhost_destroy_device_notify(dev);
 755
 756         cleanup_device(dev, 1);
 757         free_device(dev);
 758
 759         vhost_devices[vid] = NULL;
 760 }
 761
 762 void
 763 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev)
 764 {
 765         struct virtio_net *dev = get_device(vid);
 766
 767         if (dev == NULL)
 768                 return;
 769
 770         dev->vdpa_dev = vdpa_dev;
 771 }
 772
 773 void
 774 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len)
 775 {
 776         struct virtio_net *dev;
 777         unsigned int len;
 778
 779         dev = get_device(vid);
 780         if (dev == NULL)
 781                 return;
 782
 783         len = if_len > sizeof(dev->ifname) ?
 784                 sizeof(dev->ifname) : if_len;
 785
 786         strncpy(dev->ifname, if_name, len);
 787         dev->ifname[sizeof(dev->ifname) - 1] = '\0';
 788 }
 789
 790 void
 791 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags, bool stats_enabled)
 792 {
 793         struct virtio_net *dev = get_device(vid);
 794
 795         if (dev == NULL)
 796                 return;
 797
 798         if (enable)
 799                 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 800         else
 801                 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 802         if (!compliant_ol_flags)
 803                 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS;
 804         else
 805                 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS;
 806         if (stats_enabled)
 807                 dev->flags |= VIRTIO_DEV_STATS_ENABLED;
 808         else
 809                 dev->flags &= ~VIRTIO_DEV_STATS_ENABLED;
 810 }
 811
 812 void
 813 vhost_enable_extbuf(int vid)
 814 {
 815         struct virtio_net *dev = get_device(vid);
 816
 817         if (dev == NULL)
 818                 return;
 819
 820         dev->extbuf = 1;
 821 }
 822
 823 void
 824 vhost_enable_linearbuf(int vid)
 825 {
 826         struct virtio_net *dev = get_device(vid);
 827
 828         if (dev == NULL)
 829                 return;
 830
 831         dev->linearbuf = 1;
 832 }
 833
 834 int
 835 rte_vhost_get_mtu(int vid, uint16_t *mtu)
 836 {
 837         struct virtio_net *dev = get_device(vid);
 838
 839         if (dev == NULL || mtu == NULL)
 840                 return -ENODEV;
 841
 842         if (!(dev->flags & VIRTIO_DEV_READY))
 843                 return -EAGAIN;
 844
 845         if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU)))
 846                 return -ENOTSUP;
 847
 848         *mtu = dev->mtu;
 849
 850         return 0;
 851 }
 852
 853 int
 854 rte_vhost_get_numa_node(int vid)
 855 {
 856 #ifdef RTE_LIBRTE_VHOST_NUMA
 857         struct virtio_net *dev = get_device(vid);
 858         int numa_node;
 859         int ret;
 860
 861         if (dev == NULL || numa_available() != 0)
 862                 return -1;
 863
 864         ret = get_mempolicy(&numa_node, NULL, 0, dev,
 865                             MPOL_F_NODE | MPOL_F_ADDR);
 866         if (ret < 0) {
 867                 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n",
 868                         dev->ifname, rte_strerror(errno));
 869                 return -1;
 870         }
 871
 872         return numa_node;
 873 #else
 874         RTE_SET_USED(vid);
 875         return -1;
 876 #endif
 877 }
 878
 879 uint32_t
 880 rte_vhost_get_queue_num(int vid)
 881 {
 882         struct virtio_net *dev = get_device(vid);
 883
 884         if (dev == NULL)
 885                 return 0;
 886
 887         return dev->nr_vring / 2;
 888 }
 889
 890 uint16_t
 891 rte_vhost_get_vring_num(int vid)
 892 {
 893         struct virtio_net *dev = get_device(vid);
 894
 895         if (dev == NULL)
 896                 return 0;
 897
 898         return dev->nr_vring;
 899 }
 900
 901 int
 902 rte_vhost_get_ifname(int vid, char *buf, size_t len)
 903 {
 904         struct virtio_net *dev = get_device(vid);
 905
 906         if (dev == NULL || buf == NULL)
 907                 return -1;
 908
 909         len = RTE_MIN(len, sizeof(dev->ifname));
 910
 911         strncpy(buf, dev->ifname, len);
 912         buf[len - 1] = '\0';
 913
 914         return 0;
 915 }
 916
 917 int
 918 rte_vhost_get_negotiated_features(int vid, uint64_t *features)
 919 {
 920         struct virtio_net *dev;
 921
 922         dev = get_device(vid);
 923         if (dev == NULL || features == NULL)
 924                 return -1;
 925
 926         *features = dev->features;
 927         return 0;
 928 }
 929
 930 int
 931 rte_vhost_get_negotiated_protocol_features(int vid,
 932                                            uint64_t *protocol_features)
 933 {
 934         struct virtio_net *dev;
 935
 936         dev = get_device(vid);
 937         if (dev == NULL || protocol_features == NULL)
 938                 return -1;
 939
 940         *protocol_features = dev->protocol_features;
 941         return 0;
 942 }
 943
 944 int
 945 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem)
 946 {
 947         struct virtio_net *dev;
 948         struct rte_vhost_memory *m;
 949         size_t size;
 950
 951         dev = get_device(vid);
 952         if (dev == NULL || mem == NULL)
 953                 return -1;
 954
 955         size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region);
 956         m = malloc(sizeof(struct rte_vhost_memory) + size);
 957         if (!m)
 958                 return -1;
 959
 960         m->nregions = dev->mem->nregions;
 961         memcpy(m->regions, dev->mem->regions, size);
 962         *mem = m;
 963
 964         return 0;
 965 }
 966
 967 int
 968 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
 969                           struct rte_vhost_vring *vring)
 970 {
 971         struct virtio_net *dev;
 972         struct vhost_virtqueue *vq;
 973
 974         dev = get_device(vid);
 975         if (dev == NULL || vring == NULL)
 976                 return -1;
 977
 978         if (vring_idx >= VHOST_MAX_VRING)
 979                 return -1;
 980
 981         vq = dev->virtqueue[vring_idx];
 982         if (!vq)
 983                 return -1;
 984
 985         if (vq_is_packed(dev)) {
 986                 vring->desc_packed = vq->desc_packed;
 987                 vring->driver_event = vq->driver_event;
 988                 vring->device_event = vq->device_event;
 989         } else {
 990                 vring->desc = vq->desc;
 991                 vring->avail = vq->avail;
 992                 vring->used = vq->used;
 993         }
 994         vring->log_guest_addr  = vq->log_guest_addr;
 995
 996         vring->callfd  = vq->callfd;
 997         vring->kickfd  = vq->kickfd;
 998         vring->size    = vq->size;
 999
1000         return 0;
1001 }
1002
1003 int
1004 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx,
1005                                   struct rte_vhost_ring_inflight *vring)
1006 {
1007         struct virtio_net *dev;
1008         struct vhost_virtqueue *vq;
1009
1010         dev = get_device(vid);
1011         if (unlikely(!dev))
1012                 return -1;
1013
1014         if (vring_idx >= VHOST_MAX_VRING)
1015                 return -1;
1016
1017         vq = dev->virtqueue[vring_idx];
1018         if (unlikely(!vq))
1019                 return -1;
1020
1021         if (vq_is_packed(dev)) {
1022                 if (unlikely(!vq->inflight_packed))
1023                         return -1;
1024
1025                 vring->inflight_packed = vq->inflight_packed;
1026         } else {
1027                 if (unlikely(!vq->inflight_split))
1028                         return -1;
1029
1030                 vring->inflight_split = vq->inflight_split;
1031         }
1032
1033         vring->resubmit_inflight = vq->resubmit_inflight;
1034
1035         return 0;
1036 }
1037
1038 int
1039 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx,
1040                                   uint16_t idx)
1041 {
1042         struct vhost_virtqueue *vq;
1043         struct virtio_net *dev;
1044
1045         dev = get_device(vid);
1046         if (unlikely(!dev))
1047                 return -1;
1048
1049         if (unlikely(!(dev->protocol_features &
1050             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1051                 return 0;
1052
1053         if (unlikely(vq_is_packed(dev)))
1054                 return -1;
1055
1056         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1057                 return -1;
1058
1059         vq = dev->virtqueue[vring_idx];
1060         if (unlikely(!vq))
1061                 return -1;
1062
1063         if (unlikely(!vq->inflight_split))
1064                 return -1;
1065
1066         if (unlikely(idx >= vq->size))
1067                 return -1;
1068
1069         vq->inflight_split->desc[idx].counter = vq->global_counter++;
1070         vq->inflight_split->desc[idx].inflight = 1;
1071         return 0;
1072 }
1073
1074 int
1075 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx,
1076                                    uint16_t head, uint16_t last,
1077                                    uint16_t *inflight_entry)
1078 {
1079         struct rte_vhost_inflight_info_packed *inflight_info;
1080         struct virtio_net *dev;
1081         struct vhost_virtqueue *vq;
1082         struct vring_packed_desc *desc;
1083         uint16_t old_free_head, free_head;
1084
1085         dev = get_device(vid);
1086         if (unlikely(!dev))
1087                 return -1;
1088
1089         if (unlikely(!(dev->protocol_features &
1090             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1091                 return 0;
1092
1093         if (unlikely(!vq_is_packed(dev)))
1094                 return -1;
1095
1096         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1097                 return -1;
1098
1099         vq = dev->virtqueue[vring_idx];
1100         if (unlikely(!vq))
1101                 return -1;
1102
1103         inflight_info = vq->inflight_packed;
1104         if (unlikely(!inflight_info))
1105                 return -1;
1106
1107         if (unlikely(head >= vq->size))
1108                 return -1;
1109
1110         desc = vq->desc_packed;
1111         old_free_head = inflight_info->old_free_head;
1112         if (unlikely(old_free_head >= vq->size))
1113                 return -1;
1114
1115         free_head = old_free_head;
1116
1117         /* init header descriptor */
1118         inflight_info->desc[old_free_head].num = 0;
1119         inflight_info->desc[old_free_head].counter = vq->global_counter++;
1120         inflight_info->desc[old_free_head].inflight = 1;
1121
1122         /* save desc entry in flight entry */
1123         while (head != ((last + 1) % vq->size)) {
1124                 inflight_info->desc[old_free_head].num++;
1125                 inflight_info->desc[free_head].addr = desc[head].addr;
1126                 inflight_info->desc[free_head].len = desc[head].len;
1127                 inflight_info->desc[free_head].flags = desc[head].flags;
1128                 inflight_info->desc[free_head].id = desc[head].id;
1129
1130                 inflight_info->desc[old_free_head].last = free_head;
1131                 free_head = inflight_info->desc[free_head].next;
1132                 inflight_info->free_head = free_head;
1133                 head = (head + 1) % vq->size;
1134         }
1135
1136         inflight_info->old_free_head = free_head;
1137         *inflight_entry = old_free_head;
1138
1139         return 0;
1140 }
1141
1142 int
1143 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
1144                                   uint16_t last_used_idx, uint16_t idx)
1145 {
1146         struct virtio_net *dev;
1147         struct vhost_virtqueue *vq;
1148
1149         dev = get_device(vid);
1150         if (unlikely(!dev))
1151                 return -1;
1152
1153         if (unlikely(!(dev->protocol_features &
1154             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1155                 return 0;
1156
1157         if (unlikely(vq_is_packed(dev)))
1158                 return -1;
1159
1160         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1161                 return -1;
1162
1163         vq = dev->virtqueue[vring_idx];
1164         if (unlikely(!vq))
1165                 return -1;
1166
1167         if (unlikely(!vq->inflight_split))
1168                 return -1;
1169
1170         if (unlikely(idx >= vq->size))
1171                 return -1;
1172
1173         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1174
1175         vq->inflight_split->desc[idx].inflight = 0;
1176
1177         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1178
1179         vq->inflight_split->used_idx = last_used_idx;
1180         return 0;
1181 }
1182
1183 int
1184 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx,
1185                                    uint16_t head)
1186 {
1187         struct rte_vhost_inflight_info_packed *inflight_info;
1188         struct virtio_net *dev;
1189         struct vhost_virtqueue *vq;
1190
1191         dev = get_device(vid);
1192         if (unlikely(!dev))
1193                 return -1;
1194
1195         if (unlikely(!(dev->protocol_features &
1196             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1197                 return 0;
1198
1199         if (unlikely(!vq_is_packed(dev)))
1200                 return -1;
1201
1202         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1203                 return -1;
1204
1205         vq = dev->virtqueue[vring_idx];
1206         if (unlikely(!vq))
1207                 return -1;
1208
1209         inflight_info = vq->inflight_packed;
1210         if (unlikely(!inflight_info))
1211                 return -1;
1212
1213         if (unlikely(head >= vq->size))
1214                 return -1;
1215
1216         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1217
1218         inflight_info->desc[head].inflight = 0;
1219
1220         rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1221
1222         inflight_info->old_free_head = inflight_info->free_head;
1223         inflight_info->old_used_idx = inflight_info->used_idx;
1224         inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter;
1225
1226         return 0;
1227 }
1228
1229 int
1230 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx,
1231                                      uint16_t idx)
1232 {
1233         struct virtio_net *dev;
1234         struct vhost_virtqueue *vq;
1235
1236         dev = get_device(vid);
1237         if (unlikely(!dev))
1238                 return -1;
1239
1240         if (unlikely(!(dev->protocol_features &
1241             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1242                 return 0;
1243
1244         if (unlikely(vq_is_packed(dev)))
1245                 return -1;
1246
1247         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1248                 return -1;
1249
1250         vq = dev->virtqueue[vring_idx];
1251         if (unlikely(!vq))
1252                 return -1;
1253
1254         if (unlikely(!vq->inflight_split))
1255                 return -1;
1256
1257         if (unlikely(idx >= vq->size))
1258                 return -1;
1259
1260         vq->inflight_split->last_inflight_io = idx;
1261         return 0;
1262 }
1263
1264 int
1265 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx,
1266                                       uint16_t head)
1267 {
1268         struct rte_vhost_inflight_info_packed *inflight_info;
1269         struct virtio_net *dev;
1270         struct vhost_virtqueue *vq;
1271         uint16_t last;
1272
1273         dev = get_device(vid);
1274         if (unlikely(!dev))
1275                 return -1;
1276
1277         if (unlikely(!(dev->protocol_features &
1278             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))))
1279                 return 0;
1280
1281         if (unlikely(!vq_is_packed(dev)))
1282                 return -1;
1283
1284         if (unlikely(vring_idx >= VHOST_MAX_VRING))
1285                 return -1;
1286
1287         vq = dev->virtqueue[vring_idx];
1288         if (unlikely(!vq))
1289                 return -1;
1290
1291         inflight_info = vq->inflight_packed;
1292         if (unlikely(!inflight_info))
1293                 return -1;
1294
1295         if (unlikely(head >= vq->size))
1296                 return -1;
1297
1298         last = inflight_info->desc[head].last;
1299         if (unlikely(last >= vq->size))
1300                 return -1;
1301
1302         inflight_info->desc[last].next = inflight_info->free_head;
1303         inflight_info->free_head = head;
1304         inflight_info->used_idx += inflight_info->desc[head].num;
1305         if (inflight_info->used_idx >= inflight_info->desc_num) {
1306                 inflight_info->used_idx -= inflight_info->desc_num;
1307                 inflight_info->used_wrap_counter =
1308                         !inflight_info->used_wrap_counter;
1309         }
1310
1311         return 0;
1312 }
1313
1314 int
1315 rte_vhost_vring_call(int vid, uint16_t vring_idx)
1316 {
1317         struct virtio_net *dev;
1318         struct vhost_virtqueue *vq;
1319
1320         dev = get_device(vid);
1321         if (!dev)
1322                 return -1;
1323
1324         if (vring_idx >= VHOST_MAX_VRING)
1325                 return -1;
1326
1327         vq = dev->virtqueue[vring_idx];
1328         if (!vq)
1329                 return -1;
1330
1331         rte_spinlock_lock(&vq->access_lock);
1332
1333         if (vq_is_packed(dev))
1334                 vhost_vring_call_packed(dev, vq);
1335         else
1336                 vhost_vring_call_split(dev, vq);
1337
1338         rte_spinlock_unlock(&vq->access_lock);
1339
1340         return 0;
1341 }
1342
1343 uint16_t
1344 rte_vhost_avail_entries(int vid, uint16_t queue_id)
1345 {
1346         struct virtio_net *dev;
1347         struct vhost_virtqueue *vq;
1348         uint16_t ret = 0;
1349
1350         dev = get_device(vid);
1351         if (!dev)
1352                 return 0;
1353
1354         if (queue_id >= VHOST_MAX_VRING)
1355                 return 0;
1356
1357         vq = dev->virtqueue[queue_id];
1358         if (!vq)
1359                 return 0;
1360
1361         rte_spinlock_lock(&vq->access_lock);
1362
1363         if (unlikely(!vq->enabled || vq->avail == NULL))
1364                 goto out;
1365
1366         ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx;
1367
1368 out:
1369         rte_spinlock_unlock(&vq->access_lock);
1370         return ret;
1371 }
1372
1373 static inline int
1374 vhost_enable_notify_split(struct virtio_net *dev,
1375                 struct vhost_virtqueue *vq, int enable)
1376 {
1377         if (vq->used == NULL)
1378                 return -1;
1379
1380         if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
1381                 if (enable)
1382                         vq->used->flags &= ~VRING_USED_F_NO_NOTIFY;
1383                 else
1384                         vq->used->flags |= VRING_USED_F_NO_NOTIFY;
1385         } else {
1386                 if (enable)
1387                         vhost_avail_event(vq) = vq->last_avail_idx;
1388         }
1389         return 0;
1390 }
1391
1392 static inline int
1393 vhost_enable_notify_packed(struct virtio_net *dev,
1394                 struct vhost_virtqueue *vq, int enable)
1395 {
1396         uint16_t flags;
1397
1398         if (vq->device_event == NULL)
1399                 return -1;
1400
1401         if (!enable) {
1402                 vq->device_event->flags = VRING_EVENT_F_DISABLE;
1403                 return 0;
1404         }
1405
1406         flags = VRING_EVENT_F_ENABLE;
1407         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
1408                 flags = VRING_EVENT_F_DESC;
1409                 vq->device_event->off_wrap = vq->last_avail_idx |
1410                         vq->avail_wrap_counter << 15;
1411         }
1412
1413         rte_atomic_thread_fence(__ATOMIC_RELEASE);
1414
1415         vq->device_event->flags = flags;
1416         return 0;
1417 }
1418
1419 int
1420 vhost_enable_guest_notification(struct virtio_net *dev,
1421                 struct vhost_virtqueue *vq, int enable)
1422 {
1423         /*
1424          * If the virtqueue is not ready yet, it will be applied
1425          * when it will become ready.
1426          */
1427         if (!vq->ready)
1428                 return 0;
1429
1430         if (vq_is_packed(dev))
1431                 return vhost_enable_notify_packed(dev, vq, enable);
1432         else
1433                 return vhost_enable_notify_split(dev, vq, enable);
1434 }
1435
1436 int
1437 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
1438 {
1439         struct virtio_net *dev = get_device(vid);
1440         struct vhost_virtqueue *vq;
1441         int ret;
1442
1443         if (!dev)
1444                 return -1;
1445
1446         if (queue_id >= VHOST_MAX_VRING)
1447                 return -1;
1448
1449         vq = dev->virtqueue[queue_id];
1450         if (!vq)
1451                 return -1;
1452
1453         rte_spinlock_lock(&vq->access_lock);
1454
1455         vq->notif_enable = enable;
1456         ret = vhost_enable_guest_notification(dev, vq, enable);
1457
1458         rte_spinlock_unlock(&vq->access_lock);
1459
1460         return ret;
1461 }
1462
1463 void
1464 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len)
1465 {
1466         struct virtio_net *dev = get_device(vid);
1467
1468         if (dev == NULL)
1469                 return;
1470
1471         vhost_log_write(dev, addr, len);
1472 }
1473
1474 void
1475 rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
1476                          uint64_t offset, uint64_t len)
1477 {
1478         struct virtio_net *dev;
1479         struct vhost_virtqueue *vq;
1480
1481         dev = get_device(vid);
1482         if (dev == NULL)
1483                 return;
1484
1485         if (vring_idx >= VHOST_MAX_VRING)
1486                 return;
1487         vq = dev->virtqueue[vring_idx];
1488         if (!vq)
1489                 return;
1490
1491         vhost_log_used_vring(dev, vq, offset, len);
1492 }
1493
1494 uint32_t
1495 rte_vhost_rx_queue_count(int vid, uint16_t qid)
1496 {
1497         struct virtio_net *dev;
1498         struct vhost_virtqueue *vq;
1499         uint32_t ret = 0;
1500
1501         dev = get_device(vid);
1502         if (dev == NULL)
1503                 return 0;
1504
1505         if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) {
1506                 VHOST_LOG_DATA(ERR, "(%s) %s: invalid virtqueue idx %d.\n",
1507                         dev->ifname, __func__, qid);
1508                 return 0;
1509         }
1510
1511         vq = dev->virtqueue[qid];
1512         if (vq == NULL)
1513                 return 0;
1514
1515         rte_spinlock_lock(&vq->access_lock);
1516
1517         if (unlikely(!vq->enabled || vq->avail == NULL))
1518                 goto out;
1519
1520         ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx;
1521
1522 out:
1523         rte_spinlock_unlock(&vq->access_lock);
1524         return ret;
1525 }
1526
1527 struct rte_vdpa_device *
1528 rte_vhost_get_vdpa_device(int vid)
1529 {
1530         struct virtio_net *dev = get_device(vid);
1531
1532         if (dev == NULL)
1533                 return NULL;
1534
1535         return dev->vdpa_dev;
1536 }
1537
1538 int
1539 rte_vhost_get_log_base(int vid, uint64_t *log_base,
1540                 uint64_t *log_size)
1541 {
1542         struct virtio_net *dev = get_device(vid);
1543
1544         if (dev == NULL || log_base == NULL || log_size == NULL)
1545                 return -1;
1546
1547         *log_base = dev->log_base;
1548         *log_size = dev->log_size;
1549
1550         return 0;
1551 }
1552
1553 int
1554 rte_vhost_get_vring_base(int vid, uint16_t queue_id,
1555                 uint16_t *last_avail_idx, uint16_t *last_used_idx)
1556 {
1557         struct vhost_virtqueue *vq;
1558         struct virtio_net *dev = get_device(vid);
1559
1560         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1561                 return -1;
1562
1563         if (queue_id >= VHOST_MAX_VRING)
1564                 return -1;
1565
1566         vq = dev->virtqueue[queue_id];
1567         if (!vq)
1568                 return -1;
1569
1570         if (vq_is_packed(dev)) {
1571                 *last_avail_idx = (vq->avail_wrap_counter << 15) |
1572                                   vq->last_avail_idx;
1573                 *last_used_idx = (vq->used_wrap_counter << 15) |
1574                                  vq->last_used_idx;
1575         } else {
1576                 *last_avail_idx = vq->last_avail_idx;
1577                 *last_used_idx = vq->last_used_idx;
1578         }
1579
1580         return 0;
1581 }
1582
1583 int
1584 rte_vhost_set_vring_base(int vid, uint16_t queue_id,
1585                 uint16_t last_avail_idx, uint16_t last_used_idx)
1586 {
1587         struct vhost_virtqueue *vq;
1588         struct virtio_net *dev = get_device(vid);
1589
1590         if (!dev)
1591                 return -1;
1592
1593         if (queue_id >= VHOST_MAX_VRING)
1594                 return -1;
1595
1596         vq = dev->virtqueue[queue_id];
1597         if (!vq)
1598                 return -1;
1599
1600         if (vq_is_packed(dev)) {
1601                 vq->last_avail_idx = last_avail_idx & 0x7fff;
1602                 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15));
1603                 vq->last_used_idx = last_used_idx & 0x7fff;
1604                 vq->used_wrap_counter = !!(last_used_idx & (1 << 15));
1605         } else {
1606                 vq->last_avail_idx = last_avail_idx;
1607                 vq->last_used_idx = last_used_idx;
1608         }
1609
1610         return 0;
1611 }
1612
1613 int
1614 rte_vhost_get_vring_base_from_inflight(int vid,
1615                                        uint16_t queue_id,
1616                                        uint16_t *last_avail_idx,
1617                                        uint16_t *last_used_idx)
1618 {
1619         struct rte_vhost_inflight_info_packed *inflight_info;
1620         struct vhost_virtqueue *vq;
1621         struct virtio_net *dev = get_device(vid);
1622
1623         if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
1624                 return -1;
1625
1626         if (queue_id >= VHOST_MAX_VRING)
1627                 return -1;
1628
1629         vq = dev->virtqueue[queue_id];
1630         if (!vq)
1631                 return -1;
1632
1633         if (!vq_is_packed(dev))
1634                 return -1;
1635
1636         inflight_info = vq->inflight_packed;
1637         if (!inflight_info)
1638                 return -1;
1639
1640         *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) |
1641                           inflight_info->old_used_idx;
1642         *last_used_idx = *last_avail_idx;
1643
1644         return 0;
1645 }
1646
1647 int
1648 rte_vhost_extern_callback_register(int vid,
1649                 struct rte_vhost_user_extern_ops const * const ops, void *ctx)
1650 {
1651         struct virtio_net *dev = get_device(vid);
1652
1653         if (dev == NULL || ops == NULL)
1654                 return -1;
1655
1656         dev->extern_ops = *ops;
1657         dev->extern_data = ctx;
1658         return 0;
1659 }
1660
1661 static __rte_always_inline int
1662 async_channel_register(int vid, uint16_t queue_id)
1663 {
1664         struct virtio_net *dev = get_device(vid);
1665         struct vhost_virtqueue *vq = dev->virtqueue[queue_id];
1666         struct vhost_async *async;
1667         int node = vq->numa_node;
1668
1669         if (unlikely(vq->async)) {
1670                 VHOST_LOG_CONFIG(ERR,
1671                                 "(%s) async register failed: already registered (qid: %d)\n",
1672                                 dev->ifname, queue_id);
1673                 return -1;
1674         }
1675
1676         async = rte_zmalloc_socket(NULL, sizeof(struct vhost_async), 0, node);
1677         if (!async) {
1678                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async metadata (qid: %d)\n",
1679                                 dev->ifname, queue_id);
1680                 return -1;
1681         }
1682
1683         async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info),
1684                         RTE_CACHE_LINE_SIZE, node);
1685         if (!async->pkts_info) {
1686                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async_pkts_info (qid: %d)\n",
1687                                 dev->ifname, queue_id);
1688                 goto out_free_async;
1689         }
1690
1691         async->pkts_cmpl_flag = rte_zmalloc_socket(NULL, vq->size * sizeof(bool),
1692                         RTE_CACHE_LINE_SIZE, node);
1693         if (!async->pkts_cmpl_flag) {
1694                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async pkts_cmpl_flag (qid: %d)\n",
1695                                 dev->ifname, queue_id);
1696                 goto out_free_async;
1697         }
1698
1699         if (vq_is_packed(dev)) {
1700                 async->buffers_packed = rte_malloc_socket(NULL,
1701                                 vq->size * sizeof(struct vring_used_elem_packed),
1702                                 RTE_CACHE_LINE_SIZE, node);
1703                 if (!async->buffers_packed) {
1704                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async buffers (qid: %d)\n",
1705                                         dev->ifname, queue_id);
1706                         goto out_free_inflight;
1707                 }
1708         } else {
1709                 async->descs_split = rte_malloc_socket(NULL,
1710                                 vq->size * sizeof(struct vring_used_elem),
1711                                 RTE_CACHE_LINE_SIZE, node);
1712                 if (!async->descs_split) {
1713                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async descs (qid: %d)\n",
1714                                         dev->ifname, queue_id);
1715                         goto out_free_inflight;
1716                 }
1717         }
1718
1719         vq->async = async;
1720
1721         return 0;
1722 out_free_inflight:
1723         rte_free(async->pkts_info);
1724 out_free_async:
1725         rte_free(async);
1726
1727         return -1;
1728 }
1729
1730 int
1731 rte_vhost_async_channel_register(int vid, uint16_t queue_id)
1732 {
1733         struct vhost_virtqueue *vq;
1734         struct virtio_net *dev = get_device(vid);
1735         int ret;
1736
1737         if (dev == NULL)
1738                 return -1;
1739
1740         if (queue_id >= VHOST_MAX_VRING)
1741                 return -1;
1742
1743         vq = dev->virtqueue[queue_id];
1744
1745         if (unlikely(vq == NULL || !dev->async_copy))
1746                 return -1;
1747
1748         rte_spinlock_lock(&vq->access_lock);
1749         ret = async_channel_register(vid, queue_id);
1750         rte_spinlock_unlock(&vq->access_lock);
1751
1752         return ret;
1753 }
1754
1755 int
1756 rte_vhost_async_channel_register_thread_unsafe(int vid, uint16_t queue_id)
1757 {
1758         struct vhost_virtqueue *vq;
1759         struct virtio_net *dev = get_device(vid);
1760
1761         if (dev == NULL)
1762                 return -1;
1763
1764         if (queue_id >= VHOST_MAX_VRING)
1765                 return -1;
1766
1767         vq = dev->virtqueue[queue_id];
1768
1769         if (unlikely(vq == NULL || !dev->async_copy))
1770                 return -1;
1771
1772         if (unlikely(!rte_spinlock_is_locked(&vq->access_lock))) {
1773                 VHOST_LOG_CONFIG(ERR, "(%s) %s() called without access lock taken.\n",
1774                                 dev->ifname, __func__);
1775                 return -1;
1776         }
1777
1778         return async_channel_register(vid, queue_id);
1779 }
1780
1781 int
1782 rte_vhost_async_channel_unregister(int vid, uint16_t queue_id)
1783 {
1784         struct vhost_virtqueue *vq;
1785         struct virtio_net *dev = get_device(vid);
1786         int ret = -1;
1787
1788         if (dev == NULL)
1789                 return ret;
1790
1791         if (queue_id >= VHOST_MAX_VRING)
1792                 return ret;
1793
1794         vq = dev->virtqueue[queue_id];
1795
1796         if (vq == NULL)
1797                 return ret;
1798
1799         if (!rte_spinlock_trylock(&vq->access_lock)) {
1800                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel, virtqueue busy.\n",
1801                                 dev->ifname);
1802                 return ret;
1803         }
1804
1805         if (!vq->async) {
1806                 ret = 0;
1807         } else if (vq->async->pkts_inflight_n) {
1808                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname);
1809                 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n",
1810                         dev->ifname);
1811         } else {
1812                 vhost_free_async_mem(vq);
1813                 ret = 0;
1814         }
1815
1816         rte_spinlock_unlock(&vq->access_lock);
1817
1818         return ret;
1819 }
1820
1821 int
1822 rte_vhost_async_channel_unregister_thread_unsafe(int vid, uint16_t queue_id)
1823 {
1824         struct vhost_virtqueue *vq;
1825         struct virtio_net *dev = get_device(vid);
1826
1827         if (dev == NULL)
1828                 return -1;
1829
1830         if (queue_id >= VHOST_MAX_VRING)
1831                 return -1;
1832
1833         vq = dev->virtqueue[queue_id];
1834
1835         if (vq == NULL)
1836                 return -1;
1837
1838         if (unlikely(!rte_spinlock_is_locked(&vq->access_lock))) {
1839                 VHOST_LOG_CONFIG(ERR, "(%s) %s() called without access lock taken.\n",
1840                                 dev->ifname, __func__);
1841                 return -1;
1842         }
1843
1844         if (!vq->async)
1845                 return 0;
1846
1847         if (vq->async->pkts_inflight_n) {
1848                 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname);
1849                 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n",
1850                         dev->ifname);
1851                 return -1;
1852         }
1853
1854         vhost_free_async_mem(vq);
1855
1856         return 0;
1857 }
1858
1859 int
1860 rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id)
1861 {
1862         struct rte_dma_info info;
1863         void *pkts_cmpl_flag_addr;
1864         uint16_t max_desc;
1865
1866         if (!rte_dma_is_valid(dma_id)) {
1867                 VHOST_LOG_CONFIG(ERR, "DMA %d is not found.\n", dma_id);
1868                 return -1;
1869         }
1870
1871         rte_dma_info_get(dma_id, &info);
1872         if (vchan_id >= info.max_vchans) {
1873                 VHOST_LOG_CONFIG(ERR, "Invalid DMA %d vChannel %u.\n", dma_id, vchan_id);
1874                 return -1;
1875         }
1876
1877         if (!dma_copy_track[dma_id].vchans) {
1878                 struct async_dma_vchan_info *vchans;
1879
1880                 vchans = rte_zmalloc(NULL, sizeof(struct async_dma_vchan_info) * info.max_vchans,
1881                                 RTE_CACHE_LINE_SIZE);
1882                 if (vchans == NULL) {
1883                         VHOST_LOG_CONFIG(ERR, "Failed to allocate vchans for DMA %d vChannel %u.\n",
1884                                         dma_id, vchan_id);
1885                         return -1;
1886                 }
1887
1888                 dma_copy_track[dma_id].vchans = vchans;
1889         }
1890
1891         if (dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) {
1892                 VHOST_LOG_CONFIG(INFO, "DMA %d vChannel %u already registered.\n", dma_id,
1893                                 vchan_id);
1894                 return 0;
1895         }
1896
1897         max_desc = info.max_desc;
1898         if (!rte_is_power_of_2(max_desc))
1899                 max_desc = rte_align32pow2(max_desc);
1900
1901         pkts_cmpl_flag_addr = rte_zmalloc(NULL, sizeof(bool *) * max_desc, RTE_CACHE_LINE_SIZE);
1902         if (!pkts_cmpl_flag_addr) {
1903                 VHOST_LOG_CONFIG(ERR, "Failed to allocate pkts_cmpl_flag_addr for DMA %d "
1904                                 "vChannel %u.\n", dma_id, vchan_id);
1905
1906                 if (dma_copy_track[dma_id].nr_vchans == 0) {
1907                         rte_free(dma_copy_track[dma_id].vchans);
1908                         dma_copy_track[dma_id].vchans = NULL;
1909                 }
1910                 return -1;
1911         }
1912
1913         dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = pkts_cmpl_flag_addr;
1914         dma_copy_track[dma_id].vchans[vchan_id].ring_size = max_desc;
1915         dma_copy_track[dma_id].vchans[vchan_id].ring_mask = max_desc - 1;
1916         dma_copy_track[dma_id].nr_vchans++;
1917
1918         return 0;
1919 }
1920
1921 int
1922 rte_vhost_async_get_inflight(int vid, uint16_t queue_id)
1923 {
1924         struct vhost_virtqueue *vq;
1925         struct virtio_net *dev = get_device(vid);
1926         int ret = -1;
1927
1928         if (dev == NULL)
1929                 return ret;
1930
1931         if (queue_id >= VHOST_MAX_VRING)
1932                 return ret;
1933
1934         vq = dev->virtqueue[queue_id];
1935
1936         if (vq == NULL)
1937                 return ret;
1938
1939         if (!rte_spinlock_trylock(&vq->access_lock)) {
1940                 VHOST_LOG_CONFIG(DEBUG,
1941                         "(%s) failed to check in-flight packets. virtqueue busy.\n",
1942                         dev->ifname);
1943                 return ret;
1944         }
1945
1946         if (vq->async)
1947                 ret = vq->async->pkts_inflight_n;
1948
1949         rte_spinlock_unlock(&vq->access_lock);
1950
1951         return ret;
1952 }
1953
1954 int
1955 rte_vhost_async_get_inflight_thread_unsafe(int vid, uint16_t queue_id)
1956 {
1957         struct vhost_virtqueue *vq;
1958         struct virtio_net *dev = get_device(vid);
1959         int ret = -1;
1960
1961         if (dev == NULL)
1962                 return ret;
1963
1964         if (queue_id >= VHOST_MAX_VRING)
1965                 return ret;
1966
1967         vq = dev->virtqueue[queue_id];
1968
1969         if (vq == NULL)
1970                 return ret;
1971
1972         if (unlikely(!rte_spinlock_is_locked(&vq->access_lock))) {
1973                 VHOST_LOG_CONFIG(ERR, "(%s) %s() called without access lock taken.\n",
1974                                 dev->ifname, __func__);
1975                 return -1;
1976         }
1977
1978         if (!vq->async)
1979                 return ret;
1980
1981         ret = vq->async->pkts_inflight_n;
1982
1983         return ret;
1984 }
1985
1986 int
1987 rte_vhost_get_monitor_addr(int vid, uint16_t queue_id,
1988                 struct rte_vhost_power_monitor_cond *pmc)
1989 {
1990         struct virtio_net *dev = get_device(vid);
1991         struct vhost_virtqueue *vq;
1992
1993         if (dev == NULL)
1994                 return -1;
1995         if (queue_id >= VHOST_MAX_VRING)
1996                 return -1;
1997
1998         vq = dev->virtqueue[queue_id];
1999         if (vq == NULL)
2000                 return -1;
2001
2002         if (vq_is_packed(dev)) {
2003                 struct vring_packed_desc *desc;
2004                 desc = vq->desc_packed;
2005                 pmc->addr = &desc[vq->last_avail_idx].flags;
2006                 if (vq->avail_wrap_counter)
2007                         pmc->val = VRING_DESC_F_AVAIL;
2008                 else
2009                         pmc->val = VRING_DESC_F_USED;
2010                 pmc->mask = VRING_DESC_F_AVAIL | VRING_DESC_F_USED;
2011                 pmc->size = sizeof(desc[vq->last_avail_idx].flags);
2012                 pmc->match = 1;
2013         } else {
2014                 pmc->addr = &vq->avail->idx;
2015                 pmc->val = vq->last_avail_idx & (vq->size - 1);
2016                 pmc->mask = vq->size - 1;
2017                 pmc->size = sizeof(vq->avail->idx);
2018                 pmc->match = 0;
2019         }
2020
2021         return 0;
2022 }
2023
2024
2025 int
2026 rte_vhost_vring_stats_get_names(int vid, uint16_t queue_id,
2027                 struct rte_vhost_stat_name *name, unsigned int size)
2028 {
2029         struct virtio_net *dev = get_device(vid);
2030         unsigned int i;
2031
2032         if (dev == NULL)
2033                 return -1;
2034
2035         if (queue_id >= dev->nr_vring)
2036                 return -1;
2037
2038         if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED))
2039                 return -1;
2040
2041         if (name == NULL || size < VHOST_NB_VQ_STATS)
2042                 return VHOST_NB_VQ_STATS;
2043
2044         for (i = 0; i < VHOST_NB_VQ_STATS; i++)
2045                 snprintf(name[i].name, sizeof(name[i].name), "%s_q%u_%s",
2046                                 (queue_id & 1) ? "rx" : "tx",
2047                                 queue_id / 2, vhost_vq_stat_strings[i].name);
2048
2049         return VHOST_NB_VQ_STATS;
2050 }
2051
2052 int
2053 rte_vhost_vring_stats_get(int vid, uint16_t queue_id,
2054                 struct rte_vhost_stat *stats, unsigned int n)
2055 {
2056         struct virtio_net *dev = get_device(vid);
2057         struct vhost_virtqueue *vq;
2058         unsigned int i;
2059
2060         if (dev == NULL)
2061                 return -1;
2062
2063         if (queue_id >= dev->nr_vring)
2064                 return -1;
2065
2066         if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED))
2067                 return -1;
2068
2069         if (stats == NULL || n < VHOST_NB_VQ_STATS)
2070                 return VHOST_NB_VQ_STATS;
2071
2072         vq = dev->virtqueue[queue_id];
2073
2074         rte_spinlock_lock(&vq->access_lock);
2075         for (i = 0; i < VHOST_NB_VQ_STATS; i++) {
2076                 stats[i].value =
2077                         *(uint64_t *)(((char *)vq) + vhost_vq_stat_strings[i].offset);
2078                 stats[i].id = i;
2079         }
2080         rte_spinlock_unlock(&vq->access_lock);
2081
2082         return VHOST_NB_VQ_STATS;
2083 }
2084
2085 int rte_vhost_vring_stats_reset(int vid, uint16_t queue_id)
2086 {
2087         struct virtio_net *dev = get_device(vid);
2088         struct vhost_virtqueue *vq;
2089
2090         if (dev == NULL)
2091                 return -1;
2092
2093         if (queue_id >= dev->nr_vring)
2094                 return -1;
2095
2096         if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED))
2097                 return -1;
2098
2099         vq = dev->virtqueue[queue_id];
2100
2101         rte_spinlock_lock(&vq->access_lock);
2102         memset(&vq->stats, 0, sizeof(vq->stats));
2103         rte_spinlock_unlock(&vq->access_lock);
2104
2105         return 0;
2106 }
2107
2108 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO);
2109 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING);