lib/vhost/vhost_user.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 /* Security model
   6  * --------------
   7  * The vhost-user protocol connection is an external interface, so it must be
   8  * robust against invalid inputs.
   9  *
  10  * This is important because the vhost-user master is only one step removed
  11  * from the guest.  Malicious guests that have escaped will then launch further
  12  * attacks from the vhost-user master.
  13  *
  14  * Even in deployments where guests are trusted, a bug in the vhost-user master
  15  * can still cause invalid messages to be sent.  Such messages must not
  16  * compromise the stability of the DPDK application by causing crashes, memory
  17  * corruption, or other problematic behavior.
  18  *
  19  * Do not assume received VhostUserMsg fields contain sensible values!
  20  */
  21
  22 #include <stdint.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28 #include <sys/ioctl.h>
  29 #include <sys/mman.h>
  30 #include <sys/stat.h>
  31 #include <sys/syscall.h>
  32 #ifdef RTE_LIBRTE_VHOST_NUMA
  33 #include <numaif.h>
  34 #endif
  35 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
  36 #include <linux/userfaultfd.h>
  37 #endif
  38 #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
  39 #include <linux/memfd.h>
  40 #define MEMFD_SUPPORTED
  41 #endif
  42
  43 #include <rte_common.h>
  44 #include <rte_malloc.h>
  45 #include <rte_log.h>
  46 #include <rte_vfio.h>
  47 #include <rte_errno.h>
  48
  49 #include "iotlb.h"
  50 #include "vhost.h"
  51 #include "vhost_user.h"
  52
  53 #define VIRTIO_MIN_MTU 68
  54 #define VIRTIO_MAX_MTU 65535
  55
  56 #define INFLIGHT_ALIGNMENT      64
  57 #define INFLIGHT_VERSION        0x1
  58
  59 typedef struct vhost_message_handler {
  60         const char *description;
  61         int (*callback)(struct virtio_net **pdev, struct vhu_msg_context *ctx,
  62                 int main_fd);
  63         bool accepts_fd;
  64 } vhost_message_handler_t;
  65 static vhost_message_handler_t vhost_message_handlers[];
  66
  67 static int send_vhost_reply(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx);
  68 static int read_vhost_message(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx);
  69
  70 static void
  71 close_msg_fds(struct vhu_msg_context *ctx)
  72 {
  73         int i;
  74
  75         for (i = 0; i < ctx->fd_num; i++) {
  76                 int fd = ctx->fds[i];
  77
  78                 if (fd == -1)
  79                         continue;
  80
  81                 ctx->fds[i] = -1;
  82                 close(fd);
  83         }
  84 }
  85
  86 /*
  87  * Ensure the expected number of FDs is received,
  88  * close all FDs and return an error if this is not the case.
  89  */
  90 static int
  91 validate_msg_fds(struct virtio_net *dev, struct vhu_msg_context *ctx, int expected_fds)
  92 {
  93         if (ctx->fd_num == expected_fds)
  94                 return 0;
  95
  96         VHOST_LOG_CONFIG(ERR, "(%s) expect %d FDs for request %s, received %d\n",
  97                 dev->ifname, expected_fds,
  98                 vhost_message_handlers[ctx->msg.request.master].description,
  99                 ctx->fd_num);
 100
 101         close_msg_fds(ctx);
 102
 103         return -1;
 104 }
 105
 106 static uint64_t
 107 get_blk_size(int fd)
 108 {
 109         struct stat stat;
 110         int ret;
 111
 112         ret = fstat(fd, &stat);
 113         return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
 114 }
 115
 116 static void
 117 async_dma_map(struct virtio_net *dev, bool do_map)
 118 {
 119         int ret = 0;
 120         uint32_t i;
 121         struct guest_page *page;
 122
 123         if (do_map) {
 124                 for (i = 0; i < dev->nr_guest_pages; i++) {
 125                         page = &dev->guest_pages[i];
 126                         ret = rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
 127                                                          page->host_user_addr,
 128                                                          page->host_iova,
 129                                                          page->size);
 130                         if (ret) {
 131                                 /*
 132                                  * DMA device may bind with kernel driver, in this case,
 133                                  * we don't need to program IOMMU manually. However, if no
 134                                  * device is bound with vfio/uio in DPDK, and vfio kernel
 135                                  * module is loaded, the API will still be called and return
 136                                  * with ENODEV.
 137                                  *
 138                                  * DPDK vfio only returns ENODEV in very similar situations
 139                                  * (vfio either unsupported, or supported but no devices found).
 140                                  * Either way, no mappings could be performed. We treat it as
 141                                  * normal case in async path. This is a workaround.
 142                                  */
 143                                 if (rte_errno == ENODEV)
 144                                         return;
 145
 146                                 /* DMA mapping errors won't stop VHOST_USER_SET_MEM_TABLE. */
 147                                 VHOST_LOG_CONFIG(ERR, "DMA engine map failed\n");
 148                         }
 149                 }
 150
 151         } else {
 152                 for (i = 0; i < dev->nr_guest_pages; i++) {
 153                         page = &dev->guest_pages[i];
 154                         ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
 155                                                            page->host_user_addr,
 156                                                            page->host_iova,
 157                                                            page->size);
 158                         if (ret) {
 159                                 /* like DMA map, ignore the kernel driver case when unmap. */
 160                                 if (rte_errno == EINVAL)
 161                                         return;
 162
 163                                 VHOST_LOG_CONFIG(ERR, "DMA engine unmap failed\n");
 164                         }
 165                 }
 166         }
 167 }
 168
 169 static void
 170 free_mem_region(struct virtio_net *dev)
 171 {
 172         uint32_t i;
 173         struct rte_vhost_mem_region *reg;
 174
 175         if (!dev || !dev->mem)
 176                 return;
 177
 178         if (dev->async_copy && rte_vfio_is_enabled("vfio"))
 179                 async_dma_map(dev, false);
 180
 181         for (i = 0; i < dev->mem->nregions; i++) {
 182                 reg = &dev->mem->regions[i];
 183                 if (reg->host_user_addr) {
 184                         munmap(reg->mmap_addr, reg->mmap_size);
 185                         close(reg->fd);
 186                 }
 187         }
 188 }
 189
 190 void
 191 vhost_backend_cleanup(struct virtio_net *dev)
 192 {
 193         struct rte_vdpa_device *vdpa_dev;
 194
 195         vdpa_dev = dev->vdpa_dev;
 196         if (vdpa_dev && vdpa_dev->ops->dev_cleanup != NULL)
 197                 vdpa_dev->ops->dev_cleanup(dev->vid);
 198
 199         if (dev->mem) {
 200                 free_mem_region(dev);
 201                 rte_free(dev->mem);
 202                 dev->mem = NULL;
 203         }
 204
 205         rte_free(dev->guest_pages);
 206         dev->guest_pages = NULL;
 207
 208         if (dev->log_addr) {
 209                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 210                 dev->log_addr = 0;
 211         }
 212
 213         if (dev->inflight_info) {
 214                 if (dev->inflight_info->addr) {
 215                         munmap(dev->inflight_info->addr,
 216                                dev->inflight_info->size);
 217                         dev->inflight_info->addr = NULL;
 218                 }
 219
 220                 if (dev->inflight_info->fd >= 0) {
 221                         close(dev->inflight_info->fd);
 222                         dev->inflight_info->fd = -1;
 223                 }
 224
 225                 rte_free(dev->inflight_info);
 226                 dev->inflight_info = NULL;
 227         }
 228
 229         if (dev->slave_req_fd >= 0) {
 230                 close(dev->slave_req_fd);
 231                 dev->slave_req_fd = -1;
 232         }
 233
 234         if (dev->postcopy_ufd >= 0) {
 235                 close(dev->postcopy_ufd);
 236                 dev->postcopy_ufd = -1;
 237         }
 238
 239         dev->postcopy_listening = 0;
 240 }
 241
 242 static void
 243 vhost_user_notify_queue_state(struct virtio_net *dev, uint16_t index,
 244                               int enable)
 245 {
 246         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
 247         struct vhost_virtqueue *vq = dev->virtqueue[index];
 248
 249         /* Configure guest notifications on enable */
 250         if (enable && vq->notif_enable != VIRTIO_UNINITIALIZED_NOTIF)
 251                 vhost_enable_guest_notification(dev, vq, vq->notif_enable);
 252
 253         if (vdpa_dev && vdpa_dev->ops->set_vring_state)
 254                 vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
 255
 256         if (dev->notify_ops->vring_state_changed)
 257                 dev->notify_ops->vring_state_changed(dev->vid,
 258                                 index, enable);
 259 }
 260
 261 /*
 262  * This function just returns success at the moment unless
 263  * the device hasn't been initialised.
 264  */
 265 static int
 266 vhost_user_set_owner(struct virtio_net **pdev __rte_unused,
 267                         struct vhu_msg_context *ctx __rte_unused,
 268                         int main_fd __rte_unused)
 269 {
 270         return RTE_VHOST_MSG_RESULT_OK;
 271 }
 272
 273 static int
 274 vhost_user_reset_owner(struct virtio_net **pdev,
 275                         struct vhu_msg_context *ctx __rte_unused,
 276                         int main_fd __rte_unused)
 277 {
 278         struct virtio_net *dev = *pdev;
 279
 280         vhost_destroy_device_notify(dev);
 281
 282         cleanup_device(dev, 0);
 283         reset_device(dev);
 284         return RTE_VHOST_MSG_RESULT_OK;
 285 }
 286
 287 /*
 288  * The features that we support are requested.
 289  */
 290 static int
 291 vhost_user_get_features(struct virtio_net **pdev,
 292                         struct vhu_msg_context *ctx,
 293                         int main_fd __rte_unused)
 294 {
 295         struct virtio_net *dev = *pdev;
 296         uint64_t features = 0;
 297
 298         rte_vhost_driver_get_features(dev->ifname, &features);
 299
 300         ctx->msg.payload.u64 = features;
 301         ctx->msg.size = sizeof(ctx->msg.payload.u64);
 302         ctx->fd_num = 0;
 303
 304         return RTE_VHOST_MSG_RESULT_REPLY;
 305 }
 306
 307 /*
 308  * The queue number that we support are requested.
 309  */
 310 static int
 311 vhost_user_get_queue_num(struct virtio_net **pdev,
 312                         struct vhu_msg_context *ctx,
 313                         int main_fd __rte_unused)
 314 {
 315         struct virtio_net *dev = *pdev;
 316         uint32_t queue_num = 0;
 317
 318         rte_vhost_driver_get_queue_num(dev->ifname, &queue_num);
 319
 320         ctx->msg.payload.u64 = (uint64_t)queue_num;
 321         ctx->msg.size = sizeof(ctx->msg.payload.u64);
 322         ctx->fd_num = 0;
 323
 324         return RTE_VHOST_MSG_RESULT_REPLY;
 325 }
 326
 327 /*
 328  * We receive the negotiated features supported by us and the virtio device.
 329  */
 330 static int
 331 vhost_user_set_features(struct virtio_net **pdev,
 332                         struct vhu_msg_context *ctx,
 333                         int main_fd __rte_unused)
 334 {
 335         struct virtio_net *dev = *pdev;
 336         uint64_t features = ctx->msg.payload.u64;
 337         uint64_t vhost_features = 0;
 338         struct rte_vdpa_device *vdpa_dev;
 339
 340         rte_vhost_driver_get_features(dev->ifname, &vhost_features);
 341         if (features & ~vhost_features) {
 342                 VHOST_LOG_CONFIG(ERR, "(%s) received invalid negotiated features.\n",
 343                         dev->ifname);
 344                 dev->flags |= VIRTIO_DEV_FEATURES_FAILED;
 345                 dev->status &= ~VIRTIO_DEVICE_STATUS_FEATURES_OK;
 346
 347                 return RTE_VHOST_MSG_RESULT_ERR;
 348         }
 349
 350         if (dev->flags & VIRTIO_DEV_RUNNING) {
 351                 if (dev->features == features)
 352                         return RTE_VHOST_MSG_RESULT_OK;
 353
 354                 /*
 355                  * Error out if master tries to change features while device is
 356                  * in running state. The exception being VHOST_F_LOG_ALL, which
 357                  * is enabled when the live-migration starts.
 358                  */
 359                 if ((dev->features ^ features) & ~(1ULL << VHOST_F_LOG_ALL)) {
 360                         VHOST_LOG_CONFIG(ERR, "(%s) features changed while device is running.\n",
 361                                 dev->ifname);
 362                         return RTE_VHOST_MSG_RESULT_ERR;
 363                 }
 364
 365                 if (dev->notify_ops->features_changed)
 366                         dev->notify_ops->features_changed(dev->vid, features);
 367         }
 368
 369         dev->features = features;
 370         if (dev->features &
 371                 ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
 372                  (1ULL << VIRTIO_F_VERSION_1) |
 373                  (1ULL << VIRTIO_F_RING_PACKED))) {
 374                 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 375         } else {
 376                 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
 377         }
 378         VHOST_LOG_CONFIG(INFO, "(%s) negotiated Virtio features: 0x%" PRIx64 "\n",
 379                         dev->ifname, dev->features);
 380         VHOST_LOG_CONFIG(DEBUG, "(%s) mergeable RX buffers %s, virtio 1 %s\n",
 381                 dev->ifname,
 382                 (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
 383                 (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
 384
 385         if ((dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) &&
 386             !(dev->features & (1ULL << VIRTIO_NET_F_MQ))) {
 387                 /*
 388                  * Remove all but first queue pair if MQ hasn't been
 389                  * negotiated. This is safe because the device is not
 390                  * running at this stage.
 391                  */
 392                 while (dev->nr_vring > 2) {
 393                         struct vhost_virtqueue *vq;
 394
 395                         vq = dev->virtqueue[--dev->nr_vring];
 396                         if (!vq)
 397                                 continue;
 398
 399                         dev->virtqueue[dev->nr_vring] = NULL;
 400                         cleanup_vq(vq, 1);
 401                         cleanup_vq_inflight(dev, vq);
 402                         free_vq(dev, vq);
 403                 }
 404         }
 405
 406         vdpa_dev = dev->vdpa_dev;
 407         if (vdpa_dev)
 408                 vdpa_dev->ops->set_features(dev->vid);
 409
 410         dev->flags &= ~VIRTIO_DEV_FEATURES_FAILED;
 411         return RTE_VHOST_MSG_RESULT_OK;
 412 }
 413
 414 /*
 415  * The virtio device sends us the size of the descriptor ring.
 416  */
 417 static int
 418 vhost_user_set_vring_num(struct virtio_net **pdev,
 419                         struct vhu_msg_context *ctx,
 420                         int main_fd __rte_unused)
 421 {
 422         struct virtio_net *dev = *pdev;
 423         struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index];
 424
 425         if (ctx->msg.payload.state.num > 32768) {
 426                 VHOST_LOG_CONFIG(ERR, "(%s) invalid virtqueue size %u\n",
 427                                 dev->ifname, ctx->msg.payload.state.num);
 428                 return RTE_VHOST_MSG_RESULT_ERR;
 429         }
 430
 431         vq->size = ctx->msg.payload.state.num;
 432
 433         /* VIRTIO 1.0, 2.4 Virtqueues says:
 434          *
 435          *   Queue Size value is always a power of 2. The maximum Queue Size
 436          *   value is 32768.
 437          *
 438          * VIRTIO 1.1 2.7 Virtqueues says:
 439          *
 440          *   Packed virtqueues support up to 2^15 entries each.
 441          */
 442         if (!vq_is_packed(dev)) {
 443                 if (vq->size & (vq->size - 1)) {
 444                         VHOST_LOG_CONFIG(ERR, "(%s) invalid virtqueue size %u\n",
 445                                         dev->ifname, vq->size);
 446                         return RTE_VHOST_MSG_RESULT_ERR;
 447                 }
 448         }
 449
 450         if (vq_is_packed(dev)) {
 451                 rte_free(vq->shadow_used_packed);
 452                 vq->shadow_used_packed = rte_malloc_socket(NULL,
 453                                 vq->size *
 454                                 sizeof(struct vring_used_elem_packed),
 455                                 RTE_CACHE_LINE_SIZE, vq->numa_node);
 456                 if (!vq->shadow_used_packed) {
 457                         VHOST_LOG_CONFIG(ERR,
 458                                 "(%s) failed to allocate memory for shadow used ring.\n",
 459                                 dev->ifname);
 460                         return RTE_VHOST_MSG_RESULT_ERR;
 461                 }
 462
 463         } else {
 464                 rte_free(vq->shadow_used_split);
 465
 466                 vq->shadow_used_split = rte_malloc_socket(NULL,
 467                                 vq->size * sizeof(struct vring_used_elem),
 468                                 RTE_CACHE_LINE_SIZE, vq->numa_node);
 469
 470                 if (!vq->shadow_used_split) {
 471                         VHOST_LOG_CONFIG(ERR,
 472                                 "(%s) failed to allocate memory for vq internal data.\n",
 473                                 dev->ifname);
 474                         return RTE_VHOST_MSG_RESULT_ERR;
 475                 }
 476         }
 477
 478         rte_free(vq->batch_copy_elems);
 479         vq->batch_copy_elems = rte_malloc_socket(NULL,
 480                                 vq->size * sizeof(struct batch_copy_elem),
 481                                 RTE_CACHE_LINE_SIZE, vq->numa_node);
 482         if (!vq->batch_copy_elems) {
 483                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate memory for batching copy.\n",
 484                         dev->ifname);
 485                 return RTE_VHOST_MSG_RESULT_ERR;
 486         }
 487
 488         return RTE_VHOST_MSG_RESULT_OK;
 489 }
 490
 491 /*
 492  * Reallocate virtio_dev, vhost_virtqueue and related data structures to
 493  * make them on the same numa node as the memory of vring descriptor.
 494  */
 495 #ifdef RTE_LIBRTE_VHOST_NUMA
 496 static struct virtio_net*
 497 numa_realloc(struct virtio_net *dev, int index)
 498 {
 499         int node, dev_node;
 500         struct virtio_net *old_dev;
 501         struct vhost_virtqueue *vq;
 502         struct batch_copy_elem *bce;
 503         struct guest_page *gp;
 504         struct rte_vhost_memory *mem;
 505         size_t mem_size;
 506         int ret;
 507
 508         old_dev = dev;
 509         vq = dev->virtqueue[index];
 510
 511         /*
 512          * If VQ is ready, it is too late to reallocate, it certainly already
 513          * happened anyway on VHOST_USER_SET_VRING_ADRR.
 514          */
 515         if (vq->ready)
 516                 return dev;
 517
 518         ret = get_mempolicy(&node, NULL, 0, vq->desc, MPOL_F_NODE | MPOL_F_ADDR);
 519         if (ret) {
 520                 VHOST_LOG_CONFIG(ERR, "(%s) unable to get virtqueue %d numa information.\n",
 521                                 dev->ifname, index);
 522                 return dev;
 523         }
 524
 525         if (node == vq->numa_node)
 526                 goto out_dev_realloc;
 527
 528         vq = rte_realloc_socket(vq, sizeof(*vq), 0, node);
 529         if (!vq) {
 530                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc virtqueue %d on node %d\n",
 531                                 dev->ifname, index, node);
 532                 return dev;
 533         }
 534
 535         if (vq != dev->virtqueue[index]) {
 536                 VHOST_LOG_CONFIG(INFO, "(%s) reallocated virtqueue on node %d\n",
 537                                 dev->ifname, node);
 538                 dev->virtqueue[index] = vq;
 539                 vhost_user_iotlb_init(dev, index);
 540         }
 541
 542         if (vq_is_packed(dev)) {
 543                 struct vring_used_elem_packed *sup;
 544
 545                 sup = rte_realloc_socket(vq->shadow_used_packed, vq->size * sizeof(*sup),
 546                                 RTE_CACHE_LINE_SIZE, node);
 547                 if (!sup) {
 548                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc shadow packed on node %d\n",
 549                                         dev->ifname, node);
 550                         return dev;
 551                 }
 552                 vq->shadow_used_packed = sup;
 553         } else {
 554                 struct vring_used_elem *sus;
 555
 556                 sus = rte_realloc_socket(vq->shadow_used_split, vq->size * sizeof(*sus),
 557                                 RTE_CACHE_LINE_SIZE, node);
 558                 if (!sus) {
 559                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc shadow split on node %d\n",
 560                                         dev->ifname, node);
 561                         return dev;
 562                 }
 563                 vq->shadow_used_split = sus;
 564         }
 565
 566         bce = rte_realloc_socket(vq->batch_copy_elems, vq->size * sizeof(*bce),
 567                         RTE_CACHE_LINE_SIZE, node);
 568         if (!bce) {
 569                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc batch copy elem on node %d\n",
 570                                 dev->ifname, node);
 571                 return dev;
 572         }
 573         vq->batch_copy_elems = bce;
 574
 575         if (vq->log_cache) {
 576                 struct log_cache_entry *lc;
 577
 578                 lc = rte_realloc_socket(vq->log_cache, sizeof(*lc) * VHOST_LOG_CACHE_NR, 0, node);
 579                 if (!lc) {
 580                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc log cache on node %d\n",
 581                                         dev->ifname, node);
 582                         return dev;
 583                 }
 584                 vq->log_cache = lc;
 585         }
 586
 587         if (vq->resubmit_inflight) {
 588                 struct rte_vhost_resubmit_info *ri;
 589
 590                 ri = rte_realloc_socket(vq->resubmit_inflight, sizeof(*ri), 0, node);
 591                 if (!ri) {
 592                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc resubmit inflight on node %d\n",
 593                                         dev->ifname, node);
 594                         return dev;
 595                 }
 596                 vq->resubmit_inflight = ri;
 597
 598                 if (ri->resubmit_list) {
 599                         struct rte_vhost_resubmit_desc *rd;
 600
 601                         rd = rte_realloc_socket(ri->resubmit_list, sizeof(*rd) * ri->resubmit_num,
 602                                         0, node);
 603                         if (!rd) {
 604                                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc resubmit list on node %d\n",
 605                                                 dev->ifname, node);
 606                                 return dev;
 607                         }
 608                         ri->resubmit_list = rd;
 609                 }
 610         }
 611
 612         vq->numa_node = node;
 613
 614 out_dev_realloc:
 615
 616         if (dev->flags & VIRTIO_DEV_RUNNING)
 617                 return dev;
 618
 619         ret = get_mempolicy(&dev_node, NULL, 0, dev, MPOL_F_NODE | MPOL_F_ADDR);
 620         if (ret) {
 621                 VHOST_LOG_CONFIG(ERR, "(%s) unable to get numa information.\n", dev->ifname);
 622                 return dev;
 623         }
 624
 625         if (dev_node == node)
 626                 return dev;
 627
 628         dev = rte_realloc_socket(old_dev, sizeof(*dev), 0, node);
 629         if (!dev) {
 630                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc dev on node %d\n",
 631                                 old_dev->ifname, node);
 632                 return old_dev;
 633         }
 634
 635         VHOST_LOG_CONFIG(INFO, "(%s) reallocated device on node %d\n", dev->ifname, node);
 636         vhost_devices[dev->vid] = dev;
 637
 638         mem_size = sizeof(struct rte_vhost_memory) +
 639                 sizeof(struct rte_vhost_mem_region) * dev->mem->nregions;
 640         mem = rte_realloc_socket(dev->mem, mem_size, 0, node);
 641         if (!mem) {
 642                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc mem table on node %d\n",
 643                                 dev->ifname, node);
 644                 return dev;
 645         }
 646         dev->mem = mem;
 647
 648         gp = rte_realloc_socket(dev->guest_pages, dev->max_guest_pages * sizeof(*gp),
 649                         RTE_CACHE_LINE_SIZE, node);
 650         if (!gp) {
 651                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc guest pages on node %d\n",
 652                                 dev->ifname, node);
 653                 return dev;
 654         }
 655         dev->guest_pages = gp;
 656
 657         return dev;
 658 }
 659 #else
 660 static struct virtio_net*
 661 numa_realloc(struct virtio_net *dev, int index __rte_unused)
 662 {
 663         return dev;
 664 }
 665 #endif
 666
 667 /* Converts QEMU virtual address to Vhost virtual address. */
 668 static uint64_t
 669 qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
 670 {
 671         struct rte_vhost_mem_region *r;
 672         uint32_t i;
 673
 674         if (unlikely(!dev || !dev->mem))
 675                 goto out_error;
 676
 677         /* Find the region where the address lives. */
 678         for (i = 0; i < dev->mem->nregions; i++) {
 679                 r = &dev->mem->regions[i];
 680
 681                 if (qva >= r->guest_user_addr &&
 682                     qva <  r->guest_user_addr + r->size) {
 683
 684                         if (unlikely(*len > r->guest_user_addr + r->size - qva))
 685                                 *len = r->guest_user_addr + r->size - qva;
 686
 687                         return qva - r->guest_user_addr +
 688                                r->host_user_addr;
 689                 }
 690         }
 691 out_error:
 692         *len = 0;
 693
 694         return 0;
 695 }
 696
 697
 698 /*
 699  * Converts ring address to Vhost virtual address.
 700  * If IOMMU is enabled, the ring address is a guest IO virtual address,
 701  * else it is a QEMU virtual address.
 702  */
 703 static uint64_t
 704 ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 705                 uint64_t ra, uint64_t *size)
 706 {
 707         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 708                 uint64_t vva;
 709
 710                 vhost_user_iotlb_rd_lock(vq);
 711                 vva = vhost_iova_to_vva(dev, vq, ra,
 712                                         size, VHOST_ACCESS_RW);
 713                 vhost_user_iotlb_rd_unlock(vq);
 714
 715                 return vva;
 716         }
 717
 718         return qva_to_vva(dev, ra, size);
 719 }
 720
 721 static uint64_t
 722 log_addr_to_gpa(struct virtio_net *dev, struct vhost_virtqueue *vq)
 723 {
 724         uint64_t log_gpa;
 725
 726         vhost_user_iotlb_rd_lock(vq);
 727         log_gpa = translate_log_addr(dev, vq, vq->ring_addrs.log_guest_addr);
 728         vhost_user_iotlb_rd_unlock(vq);
 729
 730         return log_gpa;
 731 }
 732
 733 static struct virtio_net *
 734 translate_ring_addresses(struct virtio_net *dev, int vq_index)
 735 {
 736         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 737         struct vhost_vring_addr *addr = &vq->ring_addrs;
 738         uint64_t len, expected_len;
 739
 740         if (addr->flags & (1 << VHOST_VRING_F_LOG)) {
 741                 vq->log_guest_addr =
 742                         log_addr_to_gpa(dev, vq);
 743                 if (vq->log_guest_addr == 0) {
 744                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map log_guest_addr.\n",
 745                                 dev->ifname);
 746                         return dev;
 747                 }
 748         }
 749
 750         if (vq_is_packed(dev)) {
 751                 len = sizeof(struct vring_packed_desc) * vq->size;
 752                 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 753                         ring_addr_to_vva(dev, vq, addr->desc_user_addr, &len);
 754                 if (vq->desc_packed == NULL ||
 755                                 len != sizeof(struct vring_packed_desc) *
 756                                 vq->size) {
 757                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map desc_packed ring.\n",
 758                                 dev->ifname);
 759                         return dev;
 760                 }
 761
 762                 dev = numa_realloc(dev, vq_index);
 763                 vq = dev->virtqueue[vq_index];
 764                 addr = &vq->ring_addrs;
 765
 766                 len = sizeof(struct vring_packed_desc_event);
 767                 vq->driver_event = (struct vring_packed_desc_event *)
 768                                         (uintptr_t)ring_addr_to_vva(dev,
 769                                         vq, addr->avail_user_addr, &len);
 770                 if (vq->driver_event == NULL ||
 771                                 len != sizeof(struct vring_packed_desc_event)) {
 772                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to find driver area address.\n",
 773                                 dev->ifname);
 774                         return dev;
 775                 }
 776
 777                 len = sizeof(struct vring_packed_desc_event);
 778                 vq->device_event = (struct vring_packed_desc_event *)
 779                                         (uintptr_t)ring_addr_to_vva(dev,
 780                                         vq, addr->used_user_addr, &len);
 781                 if (vq->device_event == NULL ||
 782                                 len != sizeof(struct vring_packed_desc_event)) {
 783                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to find device area address.\n",
 784                                 dev->ifname);
 785                         return dev;
 786                 }
 787
 788                 vq->access_ok = true;
 789                 return dev;
 790         }
 791
 792         /* The addresses are converted from QEMU virtual to Vhost virtual. */
 793         if (vq->desc && vq->avail && vq->used)
 794                 return dev;
 795
 796         len = sizeof(struct vring_desc) * vq->size;
 797         vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev,
 798                         vq, addr->desc_user_addr, &len);
 799         if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) {
 800                 VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map desc ring.\n", dev->ifname);
 801                 return dev;
 802         }
 803
 804         dev = numa_realloc(dev, vq_index);
 805         vq = dev->virtqueue[vq_index];
 806         addr = &vq->ring_addrs;
 807
 808         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
 809         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 810                 len += sizeof(uint16_t);
 811         expected_len = len;
 812         vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev,
 813                         vq, addr->avail_user_addr, &len);
 814         if (vq->avail == 0 || len != expected_len) {
 815                 VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map avail ring.\n", dev->ifname);
 816                 return dev;
 817         }
 818
 819         len = sizeof(struct vring_used) +
 820                 sizeof(struct vring_used_elem) * vq->size;
 821         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 822                 len += sizeof(uint16_t);
 823         expected_len = len;
 824         vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev,
 825                         vq, addr->used_user_addr, &len);
 826         if (vq->used == 0 || len != expected_len) {
 827                 VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map used ring.\n", dev->ifname);
 828                 return dev;
 829         }
 830
 831         if (vq->last_used_idx != vq->used->idx) {
 832                 VHOST_LOG_CONFIG(WARNING, "(%s) last_used_idx (%u) and vq->used->idx (%u) mismatches;\n",
 833                         dev->ifname,
 834                         vq->last_used_idx, vq->used->idx);
 835                 vq->last_used_idx  = vq->used->idx;
 836                 vq->last_avail_idx = vq->used->idx;
 837                 VHOST_LOG_CONFIG(WARNING, "(%s) some packets maybe resent for Tx and dropped for Rx\n",
 838                         dev->ifname);
 839         }
 840
 841         vq->access_ok = true;
 842
 843         VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address desc: %p\n", dev->ifname, vq->desc);
 844         VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address avail: %p\n", dev->ifname, vq->avail);
 845         VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address used: %p\n", dev->ifname, vq->used);
 846         VHOST_LOG_CONFIG(DEBUG, "(%s) log_guest_addr: %" PRIx64 "\n",
 847                         dev->ifname, vq->log_guest_addr);
 848
 849         return dev;
 850 }
 851
 852 /*
 853  * The virtio device sends us the desc, used and avail ring addresses.
 854  * This function then converts these to our address space.
 855  */
 856 static int
 857 vhost_user_set_vring_addr(struct virtio_net **pdev,
 858                         struct vhu_msg_context *ctx,
 859                         int main_fd __rte_unused)
 860 {
 861         struct virtio_net *dev = *pdev;
 862         struct vhost_virtqueue *vq;
 863         struct vhost_vring_addr *addr = &ctx->msg.payload.addr;
 864         bool access_ok;
 865
 866         if (dev->mem == NULL)
 867                 return RTE_VHOST_MSG_RESULT_ERR;
 868
 869         /* addr->index refers to the queue index. The txq 1, rxq is 0. */
 870         vq = dev->virtqueue[ctx->msg.payload.addr.index];
 871
 872         access_ok = vq->access_ok;
 873
 874         /*
 875          * Rings addresses should not be interpreted as long as the ring is not
 876          * started and enabled
 877          */
 878         memcpy(&vq->ring_addrs, addr, sizeof(*addr));
 879
 880         vring_invalidate(dev, vq);
 881
 882         if ((vq->enabled && (dev->features &
 883                                 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) ||
 884                         access_ok) {
 885                 dev = translate_ring_addresses(dev, ctx->msg.payload.addr.index);
 886                 if (!dev)
 887                         return RTE_VHOST_MSG_RESULT_ERR;
 888
 889                 *pdev = dev;
 890         }
 891
 892         return RTE_VHOST_MSG_RESULT_OK;
 893 }
 894
 895 /*
 896  * The virtio device sends us the available ring last used index.
 897  */
 898 static int
 899 vhost_user_set_vring_base(struct virtio_net **pdev,
 900                         struct vhu_msg_context *ctx,
 901                         int main_fd __rte_unused)
 902 {
 903         struct virtio_net *dev = *pdev;
 904         struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index];
 905         uint64_t val = ctx->msg.payload.state.num;
 906
 907         if (vq_is_packed(dev)) {
 908                 /*
 909                  * Bit[0:14]: avail index
 910                  * Bit[15]: avail wrap counter
 911                  */
 912                 vq->last_avail_idx = val & 0x7fff;
 913                 vq->avail_wrap_counter = !!(val & (0x1 << 15));
 914                 /*
 915                  * Set used index to same value as available one, as
 916                  * their values should be the same since ring processing
 917                  * was stopped at get time.
 918                  */
 919                 vq->last_used_idx = vq->last_avail_idx;
 920                 vq->used_wrap_counter = vq->avail_wrap_counter;
 921         } else {
 922                 vq->last_used_idx = ctx->msg.payload.state.num;
 923                 vq->last_avail_idx = ctx->msg.payload.state.num;
 924         }
 925
 926         VHOST_LOG_CONFIG(INFO,
 927                 "(%s) vring base idx:%u last_used_idx:%u last_avail_idx:%u.\n",
 928                 dev->ifname, ctx->msg.payload.state.index, vq->last_used_idx,
 929                 vq->last_avail_idx);
 930
 931         return RTE_VHOST_MSG_RESULT_OK;
 932 }
 933
 934 static int
 935 add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
 936                    uint64_t host_iova, uint64_t host_user_addr, uint64_t size)
 937 {
 938         struct guest_page *page, *last_page;
 939         struct guest_page *old_pages;
 940
 941         if (dev->nr_guest_pages == dev->max_guest_pages) {
 942                 dev->max_guest_pages *= 2;
 943                 old_pages = dev->guest_pages;
 944                 dev->guest_pages = rte_realloc(dev->guest_pages,
 945                                         dev->max_guest_pages * sizeof(*page),
 946                                         RTE_CACHE_LINE_SIZE);
 947                 if (dev->guest_pages == NULL) {
 948                         VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
 949                         rte_free(old_pages);
 950                         return -1;
 951                 }
 952         }
 953
 954         if (dev->nr_guest_pages > 0) {
 955                 last_page = &dev->guest_pages[dev->nr_guest_pages - 1];
 956                 /* merge if the two pages are continuous */
 957                 if (host_iova == last_page->host_iova + last_page->size &&
 958                     guest_phys_addr == last_page->guest_phys_addr + last_page->size &&
 959                     host_user_addr == last_page->host_user_addr + last_page->size) {
 960                         last_page->size += size;
 961                         return 0;
 962                 }
 963         }
 964
 965         page = &dev->guest_pages[dev->nr_guest_pages++];
 966         page->guest_phys_addr = guest_phys_addr;
 967         page->host_iova  = host_iova;
 968         page->host_user_addr = host_user_addr;
 969         page->size = size;
 970
 971         return 0;
 972 }
 973
 974 static int
 975 add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
 976                 uint64_t page_size)
 977 {
 978         uint64_t reg_size = reg->size;
 979         uint64_t host_user_addr  = reg->host_user_addr;
 980         uint64_t guest_phys_addr = reg->guest_phys_addr;
 981         uint64_t host_iova;
 982         uint64_t size;
 983
 984         host_iova = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr);
 985         size = page_size - (guest_phys_addr & (page_size - 1));
 986         size = RTE_MIN(size, reg_size);
 987
 988         if (add_one_guest_page(dev, guest_phys_addr, host_iova,
 989                                host_user_addr, size) < 0)
 990                 return -1;
 991
 992         host_user_addr  += size;
 993         guest_phys_addr += size;
 994         reg_size -= size;
 995
 996         while (reg_size > 0) {
 997                 size = RTE_MIN(reg_size, page_size);
 998                 host_iova = rte_mem_virt2iova((void *)(uintptr_t)
 999                                                   host_user_addr);
1000                 if (add_one_guest_page(dev, guest_phys_addr, host_iova,
1001                                        host_user_addr, size) < 0)
1002                         return -1;
1003
1004                 host_user_addr  += size;
1005                 guest_phys_addr += size;
1006                 reg_size -= size;
1007         }
1008
1009         /* sort guest page array if over binary search threshold */
1010         if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
1011                 qsort((void *)dev->guest_pages, dev->nr_guest_pages,
1012                         sizeof(struct guest_page), guest_page_addrcmp);
1013         }
1014
1015         return 0;
1016 }
1017
1018 #ifdef RTE_LIBRTE_VHOST_DEBUG
1019 /* TODO: enable it only in debug mode? */
1020 static void
1021 dump_guest_pages(struct virtio_net *dev)
1022 {
1023         uint32_t i;
1024         struct guest_page *page;
1025
1026         for (i = 0; i < dev->nr_guest_pages; i++) {
1027                 page = &dev->guest_pages[i];
1028
1029                 VHOST_LOG_CONFIG(INFO, "(%s) guest physical page region %u\n",
1030                                 dev->ifname, i);
1031                 VHOST_LOG_CONFIG(INFO, "(%s)\tguest_phys_addr: %" PRIx64 "\n",
1032                                 dev->ifname, page->guest_phys_addr);
1033                 VHOST_LOG_CONFIG(INFO, "(%s)\thost_iova : %" PRIx64 "\n",
1034                                 dev->ifname, page->host_iova);
1035                 VHOST_LOG_CONFIG(INFO, "(%s)\tsize           : %" PRIx64 "\n",
1036                                 dev->ifname, page->size);
1037         }
1038 }
1039 #else
1040 #define dump_guest_pages(dev)
1041 #endif
1042
1043 static bool
1044 vhost_memory_changed(struct VhostUserMemory *new,
1045                      struct rte_vhost_memory *old)
1046 {
1047         uint32_t i;
1048
1049         if (new->nregions != old->nregions)
1050                 return true;
1051
1052         for (i = 0; i < new->nregions; ++i) {
1053                 VhostUserMemoryRegion *new_r = &new->regions[i];
1054                 struct rte_vhost_mem_region *old_r = &old->regions[i];
1055
1056                 if (new_r->guest_phys_addr != old_r->guest_phys_addr)
1057                         return true;
1058                 if (new_r->memory_size != old_r->size)
1059                         return true;
1060                 if (new_r->userspace_addr != old_r->guest_user_addr)
1061                         return true;
1062         }
1063
1064         return false;
1065 }
1066
1067 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
1068 static int
1069 vhost_user_postcopy_region_register(struct virtio_net *dev,
1070                 struct rte_vhost_mem_region *reg)
1071 {
1072         struct uffdio_register reg_struct;
1073
1074         /*
1075          * Let's register all the mmapped area to ensure
1076          * alignment on page boundary.
1077          */
1078         reg_struct.range.start = (uint64_t)(uintptr_t)reg->mmap_addr;
1079         reg_struct.range.len = reg->mmap_size;
1080         reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
1081
1082         if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER,
1083                                 &reg_struct)) {
1084                 VHOST_LOG_CONFIG(ERR, "(%s) failed to register ufd for region "
1085                                 "%" PRIx64 " - %" PRIx64 " (ufd = %d) %s\n",
1086                                 dev->ifname,
1087                                 (uint64_t)reg_struct.range.start,
1088                                 (uint64_t)reg_struct.range.start +
1089                                 (uint64_t)reg_struct.range.len - 1,
1090                                 dev->postcopy_ufd,
1091                                 strerror(errno));
1092                 return -1;
1093         }
1094
1095         VHOST_LOG_CONFIG(INFO,
1096                         "(%s)\t userfaultfd registered for range : %" PRIx64 " - %" PRIx64 "\n",
1097                         dev->ifname,
1098                         (uint64_t)reg_struct.range.start,
1099                         (uint64_t)reg_struct.range.start +
1100                         (uint64_t)reg_struct.range.len - 1);
1101
1102         return 0;
1103 }
1104 #else
1105 static int
1106 vhost_user_postcopy_region_register(struct virtio_net *dev __rte_unused,
1107                 struct rte_vhost_mem_region *reg __rte_unused)
1108 {
1109         return -1;
1110 }
1111 #endif
1112
1113 static int
1114 vhost_user_postcopy_register(struct virtio_net *dev, int main_fd,
1115                 struct vhu_msg_context *ctx)
1116 {
1117         struct VhostUserMemory *memory;
1118         struct rte_vhost_mem_region *reg;
1119         struct vhu_msg_context ack_ctx;
1120         uint32_t i;
1121
1122         if (!dev->postcopy_listening)
1123                 return 0;
1124
1125         /*
1126          * We haven't a better way right now than sharing
1127          * DPDK's virtual address with Qemu, so that Qemu can
1128          * retrieve the region offset when handling userfaults.
1129          */
1130         memory = &ctx->msg.payload.memory;
1131         for (i = 0; i < memory->nregions; i++) {
1132                 reg = &dev->mem->regions[i];
1133                 memory->regions[i].userspace_addr = reg->host_user_addr;
1134         }
1135
1136         /* Send the addresses back to qemu */
1137         ctx->fd_num = 0;
1138         send_vhost_reply(dev, main_fd, ctx);
1139
1140         /* Wait for qemu to acknowledge it got the addresses
1141          * we've got to wait before we're allowed to generate faults.
1142          */
1143         if (read_vhost_message(dev, main_fd, &ack_ctx) <= 0) {
1144                 VHOST_LOG_CONFIG(ERR, "(%s) failed to read qemu ack on postcopy set-mem-table\n",
1145                                 dev->ifname);
1146                 return -1;
1147         }
1148
1149         if (validate_msg_fds(dev, &ack_ctx, 0) != 0)
1150                 return -1;
1151
1152         if (ack_ctx.msg.request.master != VHOST_USER_SET_MEM_TABLE) {
1153                 VHOST_LOG_CONFIG(ERR, "(%s) bad qemu ack on postcopy set-mem-table (%d)\n",
1154                                 dev->ifname, ack_ctx.msg.request.master);
1155                 return -1;
1156         }
1157
1158         /* Now userfault register and we can use the memory */
1159         for (i = 0; i < memory->nregions; i++) {
1160                 reg = &dev->mem->regions[i];
1161                 if (vhost_user_postcopy_region_register(dev, reg) < 0)
1162                         return -1;
1163         }
1164
1165         return 0;
1166 }
1167
1168 static int
1169 vhost_user_mmap_region(struct virtio_net *dev,
1170                 struct rte_vhost_mem_region *region,
1171                 uint64_t mmap_offset)
1172 {
1173         void *mmap_addr;
1174         uint64_t mmap_size;
1175         uint64_t alignment;
1176         int populate;
1177
1178         /* Check for memory_size + mmap_offset overflow */
1179         if (mmap_offset >= -region->size) {
1180                 VHOST_LOG_CONFIG(ERR, "(%s) mmap_offset (%#"PRIx64") and memory_size (%#"PRIx64") overflow\n",
1181                                 dev->ifname, mmap_offset, region->size);
1182                 return -1;
1183         }
1184
1185         mmap_size = region->size + mmap_offset;
1186
1187         /* mmap() without flag of MAP_ANONYMOUS, should be called with length
1188          * argument aligned with hugepagesz at older longterm version Linux,
1189          * like 2.6.32 and 3.2.72, or mmap() will fail with EINVAL.
1190          *
1191          * To avoid failure, make sure in caller to keep length aligned.
1192          */
1193         alignment = get_blk_size(region->fd);
1194         if (alignment == (uint64_t)-1) {
1195                 VHOST_LOG_CONFIG(ERR, "(%s) couldn't get hugepage size through fstat\n",
1196                                 dev->ifname);
1197                 return -1;
1198         }
1199         mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
1200         if (mmap_size == 0) {
1201                 /*
1202                  * It could happen if initial mmap_size + alignment overflows
1203                  * the sizeof uint64, which could happen if either mmap_size or
1204                  * alignment value is wrong.
1205                  *
1206                  * mmap() kernel implementation would return an error, but
1207                  * better catch it before and provide useful info in the logs.
1208                  */
1209                 VHOST_LOG_CONFIG(ERR, "(%s) mmap size (0x%" PRIx64 ") or alignment (0x%" PRIx64 ") is invalid\n",
1210                                 dev->ifname, region->size + mmap_offset, alignment);
1211                 return -1;
1212         }
1213
1214         populate = dev->async_copy ? MAP_POPULATE : 0;
1215         mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
1216                         MAP_SHARED | populate, region->fd, 0);
1217
1218         if (mmap_addr == MAP_FAILED) {
1219                 VHOST_LOG_CONFIG(ERR, "(%s) mmap failed (%s).\n", dev->ifname, strerror(errno));
1220                 return -1;
1221         }
1222
1223         region->mmap_addr = mmap_addr;
1224         region->mmap_size = mmap_size;
1225         region->host_user_addr = (uint64_t)(uintptr_t)mmap_addr + mmap_offset;
1226
1227         if (dev->async_copy) {
1228                 if (add_guest_pages(dev, region, alignment) < 0) {
1229                         VHOST_LOG_CONFIG(ERR, "(%s) adding guest pages to region failed.\n",
1230                                         dev->ifname);
1231                         return -1;
1232                 }
1233         }
1234
1235         VHOST_LOG_CONFIG(INFO, "(%s) guest memory region size: 0x%" PRIx64 "\n",
1236                         dev->ifname, region->size);
1237         VHOST_LOG_CONFIG(INFO, "(%s)\t guest physical addr: 0x%" PRIx64 "\n",
1238                         dev->ifname, region->guest_phys_addr);
1239         VHOST_LOG_CONFIG(INFO, "(%s)\t guest virtual  addr: 0x%" PRIx64 "\n",
1240                         dev->ifname, region->guest_user_addr);
1241         VHOST_LOG_CONFIG(INFO, "(%s)\t host  virtual  addr: 0x%" PRIx64 "\n",
1242                         dev->ifname, region->host_user_addr);
1243         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap addr : 0x%" PRIx64 "\n",
1244                         dev->ifname, (uint64_t)(uintptr_t)mmap_addr);
1245         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap size : 0x%" PRIx64 "\n",
1246                         dev->ifname, mmap_size);
1247         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap align: 0x%" PRIx64 "\n",
1248                         dev->ifname, alignment);
1249         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap off  : 0x%" PRIx64 "\n",
1250                         dev->ifname, mmap_offset);
1251
1252         return 0;
1253 }
1254
1255 static int
1256 vhost_user_set_mem_table(struct virtio_net **pdev,
1257                         struct vhu_msg_context *ctx,
1258                         int main_fd)
1259 {
1260         struct virtio_net *dev = *pdev;
1261         struct VhostUserMemory *memory = &ctx->msg.payload.memory;
1262         struct rte_vhost_mem_region *reg;
1263         int numa_node = SOCKET_ID_ANY;
1264         uint64_t mmap_offset;
1265         uint32_t i;
1266         bool async_notify = false;
1267
1268         if (validate_msg_fds(dev, ctx, memory->nregions) != 0)
1269                 return RTE_VHOST_MSG_RESULT_ERR;
1270
1271         if (memory->nregions > VHOST_MEMORY_MAX_NREGIONS) {
1272                 VHOST_LOG_CONFIG(ERR, "(%s) too many memory regions (%u)\n",
1273                                 dev->ifname, memory->nregions);
1274                 goto close_msg_fds;
1275         }
1276
1277         if (dev->mem && !vhost_memory_changed(memory, dev->mem)) {
1278                 VHOST_LOG_CONFIG(INFO, "(%s) memory regions not changed\n", dev->ifname);
1279
1280                 close_msg_fds(ctx);
1281
1282                 return RTE_VHOST_MSG_RESULT_OK;
1283         }
1284
1285         if (dev->mem) {
1286                 if (dev->flags & VIRTIO_DEV_VDPA_CONFIGURED) {
1287                         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
1288
1289                         if (vdpa_dev && vdpa_dev->ops->dev_close)
1290                                 vdpa_dev->ops->dev_close(dev->vid);
1291                         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
1292                 }
1293
1294                 /* notify the vhost application to stop DMA transfers */
1295                 if (dev->async_copy && dev->notify_ops->vring_state_changed) {
1296                         for (i = 0; i < dev->nr_vring; i++) {
1297                                 dev->notify_ops->vring_state_changed(dev->vid,
1298                                                 i, 0);
1299                         }
1300                         async_notify = true;
1301                 }
1302
1303                 free_mem_region(dev);
1304                 rte_free(dev->mem);
1305                 dev->mem = NULL;
1306         }
1307
1308         /* Flush IOTLB cache as previous HVAs are now invalid */
1309         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1310                 for (i = 0; i < dev->nr_vring; i++)
1311                         vhost_user_iotlb_flush_all(dev->virtqueue[i]);
1312
1313         /*
1314          * If VQ 0 has already been allocated, try to allocate on the same
1315          * NUMA node. It can be reallocated later in numa_realloc().
1316          */
1317         if (dev->nr_vring > 0)
1318                 numa_node = dev->virtqueue[0]->numa_node;
1319
1320         dev->nr_guest_pages = 0;
1321         if (dev->guest_pages == NULL) {
1322                 dev->max_guest_pages = 8;
1323                 dev->guest_pages = rte_zmalloc_socket(NULL,
1324                                         dev->max_guest_pages *
1325                                         sizeof(struct guest_page),
1326                                         RTE_CACHE_LINE_SIZE,
1327                                         numa_node);
1328                 if (dev->guest_pages == NULL) {
1329                         VHOST_LOG_CONFIG(ERR,
1330                                 "(%s) failed to allocate memory for dev->guest_pages\n",
1331                                 dev->ifname);
1332                         goto close_msg_fds;
1333                 }
1334         }
1335
1336         dev->mem = rte_zmalloc_socket("vhost-mem-table", sizeof(struct rte_vhost_memory) +
1337                 sizeof(struct rte_vhost_mem_region) * memory->nregions, 0, numa_node);
1338         if (dev->mem == NULL) {
1339                 VHOST_LOG_CONFIG(ERR,
1340                         "(%s) failed to allocate memory for dev->mem\n",
1341                         dev->ifname);
1342                 goto free_guest_pages;
1343         }
1344
1345         for (i = 0; i < memory->nregions; i++) {
1346                 reg = &dev->mem->regions[i];
1347
1348                 reg->guest_phys_addr = memory->regions[i].guest_phys_addr;
1349                 reg->guest_user_addr = memory->regions[i].userspace_addr;
1350                 reg->size            = memory->regions[i].memory_size;
1351                 reg->fd              = ctx->fds[i];
1352
1353                 /*
1354                  * Assign invalid file descriptor value to avoid double
1355                  * closing on error path.
1356                  */
1357                 ctx->fds[i] = -1;
1358
1359                 mmap_offset = memory->regions[i].mmap_offset;
1360
1361                 if (vhost_user_mmap_region(dev, reg, mmap_offset) < 0) {
1362                         VHOST_LOG_CONFIG(ERR, "(%s) failed to mmap region %u\n", dev->ifname, i);
1363                         goto free_mem_table;
1364                 }
1365
1366                 dev->mem->nregions++;
1367         }
1368
1369         if (dev->async_copy && rte_vfio_is_enabled("vfio"))
1370                 async_dma_map(dev, true);
1371
1372         if (vhost_user_postcopy_register(dev, main_fd, ctx) < 0)
1373                 goto free_mem_table;
1374
1375         for (i = 0; i < dev->nr_vring; i++) {
1376                 struct vhost_virtqueue *vq = dev->virtqueue[i];
1377
1378                 if (!vq)
1379                         continue;
1380
1381                 if (vq->desc || vq->avail || vq->used) {
1382                         /*
1383                          * If the memory table got updated, the ring addresses
1384                          * need to be translated again as virtual addresses have
1385                          * changed.
1386                          */
1387                         vring_invalidate(dev, vq);
1388
1389                         dev = translate_ring_addresses(dev, i);
1390                         if (!dev) {
1391                                 dev = *pdev;
1392                                 goto free_mem_table;
1393                         }
1394
1395                         *pdev = dev;
1396                 }
1397         }
1398
1399         dump_guest_pages(dev);
1400
1401         if (async_notify) {
1402                 for (i = 0; i < dev->nr_vring; i++)
1403                         dev->notify_ops->vring_state_changed(dev->vid, i, 1);
1404         }
1405
1406         return RTE_VHOST_MSG_RESULT_OK;
1407
1408 free_mem_table:
1409         free_mem_region(dev);
1410         rte_free(dev->mem);
1411         dev->mem = NULL;
1412
1413 free_guest_pages:
1414         rte_free(dev->guest_pages);
1415         dev->guest_pages = NULL;
1416 close_msg_fds:
1417         close_msg_fds(ctx);
1418         return RTE_VHOST_MSG_RESULT_ERR;
1419 }
1420
1421 static bool
1422 vq_is_ready(struct virtio_net *dev, struct vhost_virtqueue *vq)
1423 {
1424         bool rings_ok;
1425
1426         if (!vq)
1427                 return false;
1428
1429         if (vq_is_packed(dev))
1430                 rings_ok = vq->desc_packed && vq->driver_event &&
1431                         vq->device_event;
1432         else
1433                 rings_ok = vq->desc && vq->avail && vq->used;
1434
1435         return rings_ok &&
1436                vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1437                vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1438                vq->enabled;
1439 }
1440
1441 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
1442
1443 static int
1444 virtio_is_ready(struct virtio_net *dev)
1445 {
1446         struct vhost_virtqueue *vq;
1447         uint32_t i, nr_vring = dev->nr_vring;
1448
1449         if (dev->flags & VIRTIO_DEV_READY)
1450                 return 1;
1451
1452         if (!dev->nr_vring)
1453                 return 0;
1454
1455         if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
1456                 nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
1457
1458                 if (dev->nr_vring < nr_vring)
1459                         return 0;
1460         }
1461
1462         for (i = 0; i < nr_vring; i++) {
1463                 vq = dev->virtqueue[i];
1464
1465                 if (!vq_is_ready(dev, vq))
1466                         return 0;
1467         }
1468
1469         /* If supported, ensure the frontend is really done with config */
1470         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
1471                 if (!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK))
1472                         return 0;
1473
1474         dev->flags |= VIRTIO_DEV_READY;
1475
1476         if (!(dev->flags & VIRTIO_DEV_RUNNING))
1477                 VHOST_LOG_CONFIG(INFO, "(%s) virtio is now ready for processing.\n", dev->ifname);
1478         return 1;
1479 }
1480
1481 static void *
1482 inflight_mem_alloc(struct virtio_net *dev, const char *name, size_t size, int *fd)
1483 {
1484         void *ptr;
1485         int mfd = -1;
1486         char fname[20] = "/tmp/memfd-XXXXXX";
1487
1488         *fd = -1;
1489 #ifdef MEMFD_SUPPORTED
1490         mfd = memfd_create(name, MFD_CLOEXEC);
1491 #else
1492         RTE_SET_USED(name);
1493 #endif
1494         if (mfd == -1) {
1495                 mfd = mkstemp(fname);
1496                 if (mfd == -1) {
1497                         VHOST_LOG_CONFIG(ERR, "(%s) failed to get inflight buffer fd\n",
1498                                         dev->ifname);
1499                         return NULL;
1500                 }
1501
1502                 unlink(fname);
1503         }
1504
1505         if (ftruncate(mfd, size) == -1) {
1506                 VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc inflight buffer\n", dev->ifname);
1507                 close(mfd);
1508                 return NULL;
1509         }
1510
1511         ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
1512         if (ptr == MAP_FAILED) {
1513                 VHOST_LOG_CONFIG(ERR, "(%s) failed to mmap inflight buffer\n", dev->ifname);
1514                 close(mfd);
1515                 return NULL;
1516         }
1517
1518         *fd = mfd;
1519         return ptr;
1520 }
1521
1522 static uint32_t
1523 get_pervq_shm_size_split(uint16_t queue_size)
1524 {
1525         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_split) *
1526                                   queue_size + sizeof(uint64_t) +
1527                                   sizeof(uint16_t) * 4, INFLIGHT_ALIGNMENT);
1528 }
1529
1530 static uint32_t
1531 get_pervq_shm_size_packed(uint16_t queue_size)
1532 {
1533         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_packed)
1534                                   * queue_size + sizeof(uint64_t) +
1535                                   sizeof(uint16_t) * 6 + sizeof(uint8_t) * 9,
1536                                   INFLIGHT_ALIGNMENT);
1537 }
1538
1539 static int
1540 vhost_user_get_inflight_fd(struct virtio_net **pdev,
1541                            struct vhu_msg_context *ctx,
1542                            int main_fd __rte_unused)
1543 {
1544         struct rte_vhost_inflight_info_packed *inflight_packed;
1545         uint64_t pervq_inflight_size, mmap_size;
1546         uint16_t num_queues, queue_size;
1547         struct virtio_net *dev = *pdev;
1548         int fd, i, j;
1549         int numa_node = SOCKET_ID_ANY;
1550         void *addr;
1551
1552         if (ctx->msg.size != sizeof(ctx->msg.payload.inflight)) {
1553                 VHOST_LOG_CONFIG(ERR, "(%s) invalid get_inflight_fd message size is %d\n",
1554                         dev->ifname, ctx->msg.size);
1555                 return RTE_VHOST_MSG_RESULT_ERR;
1556         }
1557
1558         /*
1559          * If VQ 0 has already been allocated, try to allocate on the same
1560          * NUMA node. It can be reallocated later in numa_realloc().
1561          */
1562         if (dev->nr_vring > 0)
1563                 numa_node = dev->virtqueue[0]->numa_node;
1564
1565         if (dev->inflight_info == NULL) {
1566                 dev->inflight_info = rte_zmalloc_socket("inflight_info",
1567                                 sizeof(struct inflight_mem_info), 0, numa_node);
1568                 if (!dev->inflight_info) {
1569                         VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc dev inflight area\n",
1570                                         dev->ifname);
1571                         return RTE_VHOST_MSG_RESULT_ERR;
1572                 }
1573                 dev->inflight_info->fd = -1;
1574         }
1575
1576         num_queues = ctx->msg.payload.inflight.num_queues;
1577         queue_size = ctx->msg.payload.inflight.queue_size;
1578
1579         VHOST_LOG_CONFIG(INFO, "(%s) get_inflight_fd num_queues: %u\n",
1580                 dev->ifname, ctx->msg.payload.inflight.num_queues);
1581         VHOST_LOG_CONFIG(INFO, "(%s) get_inflight_fd queue_size: %u\n",
1582                 dev->ifname, ctx->msg.payload.inflight.queue_size);
1583
1584         if (vq_is_packed(dev))
1585                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1586         else
1587                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1588
1589         mmap_size = num_queues * pervq_inflight_size;
1590         addr = inflight_mem_alloc(dev, "vhost-inflight", mmap_size, &fd);
1591         if (!addr) {
1592                 VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc vhost inflight area\n", dev->ifname);
1593                         ctx->msg.payload.inflight.mmap_size = 0;
1594                 return RTE_VHOST_MSG_RESULT_ERR;
1595         }
1596         memset(addr, 0, mmap_size);
1597
1598         if (dev->inflight_info->addr) {
1599                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1600                 dev->inflight_info->addr = NULL;
1601         }
1602
1603         if (dev->inflight_info->fd >= 0) {
1604                 close(dev->inflight_info->fd);
1605                 dev->inflight_info->fd = -1;
1606         }
1607
1608         dev->inflight_info->addr = addr;
1609         dev->inflight_info->size = ctx->msg.payload.inflight.mmap_size = mmap_size;
1610         dev->inflight_info->fd = ctx->fds[0] = fd;
1611         ctx->msg.payload.inflight.mmap_offset = 0;
1612         ctx->fd_num = 1;
1613
1614         if (vq_is_packed(dev)) {
1615                 for (i = 0; i < num_queues; i++) {
1616                         inflight_packed =
1617                                 (struct rte_vhost_inflight_info_packed *)addr;
1618                         inflight_packed->used_wrap_counter = 1;
1619                         inflight_packed->old_used_wrap_counter = 1;
1620                         for (j = 0; j < queue_size; j++)
1621                                 inflight_packed->desc[j].next = j + 1;
1622                         addr = (void *)((char *)addr + pervq_inflight_size);
1623                 }
1624         }
1625
1626         VHOST_LOG_CONFIG(INFO, "(%s) send inflight mmap_size: %"PRIu64"\n",
1627                         dev->ifname, ctx->msg.payload.inflight.mmap_size);
1628         VHOST_LOG_CONFIG(INFO, "(%s) send inflight mmap_offset: %"PRIu64"\n",
1629                         dev->ifname, ctx->msg.payload.inflight.mmap_offset);
1630         VHOST_LOG_CONFIG(INFO, "(%s) send inflight fd: %d\n", dev->ifname, ctx->fds[0]);
1631
1632         return RTE_VHOST_MSG_RESULT_REPLY;
1633 }
1634
1635 static int
1636 vhost_user_set_inflight_fd(struct virtio_net **pdev,
1637                            struct vhu_msg_context *ctx,
1638                            int main_fd __rte_unused)
1639 {
1640         uint64_t mmap_size, mmap_offset;
1641         uint16_t num_queues, queue_size;
1642         struct virtio_net *dev = *pdev;
1643         uint32_t pervq_inflight_size;
1644         struct vhost_virtqueue *vq;
1645         void *addr;
1646         int fd, i;
1647         int numa_node = SOCKET_ID_ANY;
1648
1649         if (validate_msg_fds(dev, ctx, 1) != 0)
1650                 return RTE_VHOST_MSG_RESULT_ERR;
1651
1652         fd = ctx->fds[0];
1653         if (ctx->msg.size != sizeof(ctx->msg.payload.inflight) || fd < 0) {
1654                 VHOST_LOG_CONFIG(ERR, "(%s) invalid set_inflight_fd message size is %d,fd is %d\n",
1655                         dev->ifname, ctx->msg.size, fd);
1656                 return RTE_VHOST_MSG_RESULT_ERR;
1657         }
1658
1659         mmap_size = ctx->msg.payload.inflight.mmap_size;
1660         mmap_offset = ctx->msg.payload.inflight.mmap_offset;
1661         num_queues = ctx->msg.payload.inflight.num_queues;
1662         queue_size = ctx->msg.payload.inflight.queue_size;
1663
1664         if (vq_is_packed(dev))
1665                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1666         else
1667                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1668
1669         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd mmap_size: %"PRIu64"\n",
1670                         dev->ifname, mmap_size);
1671         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd mmap_offset: %"PRIu64"\n",
1672                         dev->ifname, mmap_offset);
1673         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd num_queues: %u\n", dev->ifname, num_queues);
1674         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd queue_size: %u\n", dev->ifname, queue_size);
1675         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd fd: %d\n", dev->ifname, fd);
1676         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd pervq_inflight_size: %d\n",
1677                         dev->ifname, pervq_inflight_size);
1678
1679         /*
1680          * If VQ 0 has already been allocated, try to allocate on the same
1681          * NUMA node. It can be reallocated later in numa_realloc().
1682          */
1683         if (dev->nr_vring > 0)
1684                 numa_node = dev->virtqueue[0]->numa_node;
1685
1686         if (!dev->inflight_info) {
1687                 dev->inflight_info = rte_zmalloc_socket("inflight_info",
1688                                 sizeof(struct inflight_mem_info), 0, numa_node);
1689                 if (dev->inflight_info == NULL) {
1690                         VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc dev inflight area\n",
1691                                         dev->ifname);
1692                         return RTE_VHOST_MSG_RESULT_ERR;
1693                 }
1694                 dev->inflight_info->fd = -1;
1695         }
1696
1697         if (dev->inflight_info->addr) {
1698                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1699                 dev->inflight_info->addr = NULL;
1700         }
1701
1702         addr = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1703                     fd, mmap_offset);
1704         if (addr == MAP_FAILED) {
1705                 VHOST_LOG_CONFIG(ERR, "(%s) failed to mmap share memory.\n", dev->ifname);
1706                 return RTE_VHOST_MSG_RESULT_ERR;
1707         }
1708
1709         if (dev->inflight_info->fd >= 0) {
1710                 close(dev->inflight_info->fd);
1711                 dev->inflight_info->fd = -1;
1712         }
1713
1714         dev->inflight_info->fd = fd;
1715         dev->inflight_info->addr = addr;
1716         dev->inflight_info->size = mmap_size;
1717
1718         for (i = 0; i < num_queues; i++) {
1719                 vq = dev->virtqueue[i];
1720                 if (!vq)
1721                         continue;
1722
1723                 if (vq_is_packed(dev)) {
1724                         vq->inflight_packed = addr;
1725                         vq->inflight_packed->desc_num = queue_size;
1726                 } else {
1727                         vq->inflight_split = addr;
1728                         vq->inflight_split->desc_num = queue_size;
1729                 }
1730                 addr = (void *)((char *)addr + pervq_inflight_size);
1731         }
1732
1733         return RTE_VHOST_MSG_RESULT_OK;
1734 }
1735
1736 static int
1737 vhost_user_set_vring_call(struct virtio_net **pdev,
1738                         struct vhu_msg_context *ctx,
1739                         int main_fd __rte_unused)
1740 {
1741         struct virtio_net *dev = *pdev;
1742         struct vhost_vring_file file;
1743         struct vhost_virtqueue *vq;
1744         int expected_fds;
1745
1746         expected_fds = (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1747         if (validate_msg_fds(dev, ctx, expected_fds) != 0)
1748                 return RTE_VHOST_MSG_RESULT_ERR;
1749
1750         file.index = ctx->msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
1751         if (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1752                 file.fd = VIRTIO_INVALID_EVENTFD;
1753         else
1754                 file.fd = ctx->fds[0];
1755         VHOST_LOG_CONFIG(INFO, "(%s) vring call idx:%d file:%d\n",
1756                         dev->ifname, file.index, file.fd);
1757
1758         vq = dev->virtqueue[file.index];
1759
1760         if (vq->ready) {
1761                 vq->ready = false;
1762                 vhost_user_notify_queue_state(dev, file.index, 0);
1763         }
1764
1765         if (vq->callfd >= 0)
1766                 close(vq->callfd);
1767
1768         vq->callfd = file.fd;
1769
1770         return RTE_VHOST_MSG_RESULT_OK;
1771 }
1772
1773 static int vhost_user_set_vring_err(struct virtio_net **pdev,
1774                         struct vhu_msg_context *ctx,
1775                         int main_fd __rte_unused)
1776 {
1777         struct virtio_net *dev = *pdev;
1778         int expected_fds;
1779
1780         expected_fds = (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1781         if (validate_msg_fds(dev, ctx, expected_fds) != 0)
1782                 return RTE_VHOST_MSG_RESULT_ERR;
1783
1784         if (!(ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1785                 close(ctx->fds[0]);
1786         VHOST_LOG_CONFIG(INFO, "(%s) not implemented\n", dev->ifname);
1787
1788         return RTE_VHOST_MSG_RESULT_OK;
1789 }
1790
1791 static int
1792 resubmit_desc_compare(const void *a, const void *b)
1793 {
1794         const struct rte_vhost_resubmit_desc *desc0 = a;
1795         const struct rte_vhost_resubmit_desc *desc1 = b;
1796
1797         if (desc1->counter > desc0->counter)
1798                 return 1;
1799
1800         return -1;
1801 }
1802
1803 static int
1804 vhost_check_queue_inflights_split(struct virtio_net *dev,
1805                                   struct vhost_virtqueue *vq)
1806 {
1807         uint16_t i;
1808         uint16_t resubmit_num = 0, last_io, num;
1809         struct vring_used *used = vq->used;
1810         struct rte_vhost_resubmit_info *resubmit;
1811         struct rte_vhost_inflight_info_split *inflight_split;
1812
1813         if (!(dev->protocol_features &
1814             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1815                 return RTE_VHOST_MSG_RESULT_OK;
1816
1817         /* The frontend may still not support the inflight feature
1818          * although we negotiate the protocol feature.
1819          */
1820         if ((!vq->inflight_split))
1821                 return RTE_VHOST_MSG_RESULT_OK;
1822
1823         if (!vq->inflight_split->version) {
1824                 vq->inflight_split->version = INFLIGHT_VERSION;
1825                 return RTE_VHOST_MSG_RESULT_OK;
1826         }
1827
1828         if (vq->resubmit_inflight)
1829                 return RTE_VHOST_MSG_RESULT_OK;
1830
1831         inflight_split = vq->inflight_split;
1832         vq->global_counter = 0;
1833         last_io = inflight_split->last_inflight_io;
1834
1835         if (inflight_split->used_idx != used->idx) {
1836                 inflight_split->desc[last_io].inflight = 0;
1837                 rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1838                 inflight_split->used_idx = used->idx;
1839         }
1840
1841         for (i = 0; i < inflight_split->desc_num; i++) {
1842                 if (inflight_split->desc[i].inflight == 1)
1843                         resubmit_num++;
1844         }
1845
1846         vq->last_avail_idx += resubmit_num;
1847
1848         if (resubmit_num) {
1849                 resubmit = rte_zmalloc_socket("resubmit", sizeof(struct rte_vhost_resubmit_info),
1850                                 0, vq->numa_node);
1851                 if (!resubmit) {
1852                         VHOST_LOG_CONFIG(ERR,
1853                                         "(%s) failed to allocate memory for resubmit info.\n",
1854                                         dev->ifname);
1855                         return RTE_VHOST_MSG_RESULT_ERR;
1856                 }
1857
1858                 resubmit->resubmit_list = rte_zmalloc_socket("resubmit_list",
1859                                 resubmit_num * sizeof(struct rte_vhost_resubmit_desc),
1860                                 0, vq->numa_node);
1861                 if (!resubmit->resubmit_list) {
1862                         VHOST_LOG_CONFIG(ERR,
1863                                         "(%s) failed to allocate memory for inflight desc.\n",
1864                                         dev->ifname);
1865                         rte_free(resubmit);
1866                         return RTE_VHOST_MSG_RESULT_ERR;
1867                 }
1868
1869                 num = 0;
1870                 for (i = 0; i < vq->inflight_split->desc_num; i++) {
1871                         if (vq->inflight_split->desc[i].inflight == 1) {
1872                                 resubmit->resubmit_list[num].index = i;
1873                                 resubmit->resubmit_list[num].counter =
1874                                         inflight_split->desc[i].counter;
1875                                 num++;
1876                         }
1877                 }
1878                 resubmit->resubmit_num = num;
1879
1880                 if (resubmit->resubmit_num > 1)
1881                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1882                               sizeof(struct rte_vhost_resubmit_desc),
1883                               resubmit_desc_compare);
1884
1885                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1886                 vq->resubmit_inflight = resubmit;
1887         }
1888
1889         return RTE_VHOST_MSG_RESULT_OK;
1890 }
1891
1892 static int
1893 vhost_check_queue_inflights_packed(struct virtio_net *dev,
1894                                    struct vhost_virtqueue *vq)
1895 {
1896         uint16_t i;
1897         uint16_t resubmit_num = 0, old_used_idx, num;
1898         struct rte_vhost_resubmit_info *resubmit;
1899         struct rte_vhost_inflight_info_packed *inflight_packed;
1900
1901         if (!(dev->protocol_features &
1902             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1903                 return RTE_VHOST_MSG_RESULT_OK;
1904
1905         /* The frontend may still not support the inflight feature
1906          * although we negotiate the protocol feature.
1907          */
1908         if ((!vq->inflight_packed))
1909                 return RTE_VHOST_MSG_RESULT_OK;
1910
1911         if (!vq->inflight_packed->version) {
1912                 vq->inflight_packed->version = INFLIGHT_VERSION;
1913                 return RTE_VHOST_MSG_RESULT_OK;
1914         }
1915
1916         if (vq->resubmit_inflight)
1917                 return RTE_VHOST_MSG_RESULT_OK;
1918
1919         inflight_packed = vq->inflight_packed;
1920         vq->global_counter = 0;
1921         old_used_idx = inflight_packed->old_used_idx;
1922
1923         if (inflight_packed->used_idx != old_used_idx) {
1924                 if (inflight_packed->desc[old_used_idx].inflight == 0) {
1925                         inflight_packed->old_used_idx =
1926                                 inflight_packed->used_idx;
1927                         inflight_packed->old_used_wrap_counter =
1928                                 inflight_packed->used_wrap_counter;
1929                         inflight_packed->old_free_head =
1930                                 inflight_packed->free_head;
1931                 } else {
1932                         inflight_packed->used_idx =
1933                                 inflight_packed->old_used_idx;
1934                         inflight_packed->used_wrap_counter =
1935                                 inflight_packed->old_used_wrap_counter;
1936                         inflight_packed->free_head =
1937                                 inflight_packed->old_free_head;
1938                 }
1939         }
1940
1941         for (i = 0; i < inflight_packed->desc_num; i++) {
1942                 if (inflight_packed->desc[i].inflight == 1)
1943                         resubmit_num++;
1944         }
1945
1946         if (resubmit_num) {
1947                 resubmit = rte_zmalloc_socket("resubmit", sizeof(struct rte_vhost_resubmit_info),
1948                                 0, vq->numa_node);
1949                 if (resubmit == NULL) {
1950                         VHOST_LOG_CONFIG(ERR,
1951                                         "(%s) failed to allocate memory for resubmit info.\n",
1952                                         dev->ifname);
1953                         return RTE_VHOST_MSG_RESULT_ERR;
1954                 }
1955
1956                 resubmit->resubmit_list = rte_zmalloc_socket("resubmit_list",
1957                                 resubmit_num * sizeof(struct rte_vhost_resubmit_desc),
1958                                 0, vq->numa_node);
1959                 if (resubmit->resubmit_list == NULL) {
1960                         VHOST_LOG_CONFIG(ERR,
1961                                         "(%s) failed to allocate memory for resubmit desc.\n",
1962                                         dev->ifname);
1963                         rte_free(resubmit);
1964                         return RTE_VHOST_MSG_RESULT_ERR;
1965                 }
1966
1967                 num = 0;
1968                 for (i = 0; i < inflight_packed->desc_num; i++) {
1969                         if (vq->inflight_packed->desc[i].inflight == 1) {
1970                                 resubmit->resubmit_list[num].index = i;
1971                                 resubmit->resubmit_list[num].counter =
1972                                         inflight_packed->desc[i].counter;
1973                                 num++;
1974                         }
1975                 }
1976                 resubmit->resubmit_num = num;
1977
1978                 if (resubmit->resubmit_num > 1)
1979                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1980                               sizeof(struct rte_vhost_resubmit_desc),
1981                               resubmit_desc_compare);
1982
1983                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1984                 vq->resubmit_inflight = resubmit;
1985         }
1986
1987         return RTE_VHOST_MSG_RESULT_OK;
1988 }
1989
1990 static int
1991 vhost_user_set_vring_kick(struct virtio_net **pdev,
1992                         struct vhu_msg_context *ctx,
1993                         int main_fd __rte_unused)
1994 {
1995         struct virtio_net *dev = *pdev;
1996         struct vhost_vring_file file;
1997         struct vhost_virtqueue *vq;
1998         int expected_fds;
1999
2000         expected_fds = (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
2001         if (validate_msg_fds(dev, ctx, expected_fds) != 0)
2002                 return RTE_VHOST_MSG_RESULT_ERR;
2003
2004         file.index = ctx->msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
2005         if (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)
2006                 file.fd = VIRTIO_INVALID_EVENTFD;
2007         else
2008                 file.fd = ctx->fds[0];
2009         VHOST_LOG_CONFIG(INFO, "(%s) vring kick idx:%d file:%d\n",
2010                         dev->ifname, file.index, file.fd);
2011
2012         /* Interpret ring addresses only when ring is started. */
2013         dev = translate_ring_addresses(dev, file.index);
2014         if (!dev) {
2015                 if (file.fd != VIRTIO_INVALID_EVENTFD)
2016                         close(file.fd);
2017
2018                 return RTE_VHOST_MSG_RESULT_ERR;
2019         }
2020
2021         *pdev = dev;
2022
2023         vq = dev->virtqueue[file.index];
2024
2025         /*
2026          * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated,
2027          * the ring starts already enabled. Otherwise, it is enabled via
2028          * the SET_VRING_ENABLE message.
2029          */
2030         if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) {
2031                 vq->enabled = true;
2032         }
2033
2034         if (vq->ready) {
2035                 vq->ready = false;
2036                 vhost_user_notify_queue_state(dev, file.index, 0);
2037         }
2038
2039         if (vq->kickfd >= 0)
2040                 close(vq->kickfd);
2041         vq->kickfd = file.fd;
2042
2043         if (vq_is_packed(dev)) {
2044                 if (vhost_check_queue_inflights_packed(dev, vq)) {
2045                         VHOST_LOG_CONFIG(ERR, "(%s) failed to inflights for vq: %d\n",
2046                                         dev->ifname, file.index);
2047                         return RTE_VHOST_MSG_RESULT_ERR;
2048                 }
2049         } else {
2050                 if (vhost_check_queue_inflights_split(dev, vq)) {
2051                         VHOST_LOG_CONFIG(ERR, "(%s) failed to inflights for vq: %d\n",
2052                                         dev->ifname, file.index);
2053                         return RTE_VHOST_MSG_RESULT_ERR;
2054                 }
2055         }
2056
2057         return RTE_VHOST_MSG_RESULT_OK;
2058 }
2059
2060 /*
2061  * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
2062  */
2063 static int
2064 vhost_user_get_vring_base(struct virtio_net **pdev,
2065                         struct vhu_msg_context *ctx,
2066                         int main_fd __rte_unused)
2067 {
2068         struct virtio_net *dev = *pdev;
2069         struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index];
2070         uint64_t val;
2071
2072         /* We have to stop the queue (virtio) if it is running. */
2073         vhost_destroy_device_notify(dev);
2074
2075         dev->flags &= ~VIRTIO_DEV_READY;
2076         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
2077
2078         /* Here we are safe to get the indexes */
2079         if (vq_is_packed(dev)) {
2080                 /*
2081                  * Bit[0:14]: avail index
2082                  * Bit[15]: avail wrap counter
2083                  */
2084                 val = vq->last_avail_idx & 0x7fff;
2085                 val |= vq->avail_wrap_counter << 15;
2086                 ctx->msg.payload.state.num = val;
2087         } else {
2088                 ctx->msg.payload.state.num = vq->last_avail_idx;
2089         }
2090
2091         VHOST_LOG_CONFIG(INFO, "(%s) vring base idx:%d file:%d\n",
2092                         dev->ifname, ctx->msg.payload.state.index,
2093                         ctx->msg.payload.state.num);
2094         /*
2095          * Based on current qemu vhost-user implementation, this message is
2096          * sent and only sent in vhost_vring_stop.
2097          * TODO: cleanup the vring, it isn't usable since here.
2098          */
2099         if (vq->kickfd >= 0)
2100                 close(vq->kickfd);
2101
2102         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
2103
2104         if (vq->callfd >= 0)
2105                 close(vq->callfd);
2106
2107         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
2108
2109         vq->signalled_used_valid = false;
2110
2111         if (vq_is_packed(dev)) {
2112                 rte_free(vq->shadow_used_packed);
2113                 vq->shadow_used_packed = NULL;
2114         } else {
2115                 rte_free(vq->shadow_used_split);
2116                 vq->shadow_used_split = NULL;
2117         }
2118
2119         rte_free(vq->batch_copy_elems);
2120         vq->batch_copy_elems = NULL;
2121
2122         rte_free(vq->log_cache);
2123         vq->log_cache = NULL;
2124
2125         ctx->msg.size = sizeof(ctx->msg.payload.state);
2126         ctx->fd_num = 0;
2127
2128         vhost_user_iotlb_flush_all(vq);
2129
2130         vring_invalidate(dev, vq);
2131
2132         return RTE_VHOST_MSG_RESULT_REPLY;
2133 }
2134
2135 /*
2136  * when virtio queues are ready to work, qemu will send us to
2137  * enable the virtio queue pair.
2138  */
2139 static int
2140 vhost_user_set_vring_enable(struct virtio_net **pdev,
2141                         struct vhu_msg_context *ctx,
2142                         int main_fd __rte_unused)
2143 {
2144         struct virtio_net *dev = *pdev;
2145         bool enable = !!ctx->msg.payload.state.num;
2146         int index = (int)ctx->msg.payload.state.index;
2147
2148         VHOST_LOG_CONFIG(INFO, "(%s) set queue enable: %d to qp idx: %d\n",
2149                         dev->ifname, enable, index);
2150
2151         if (enable && dev->virtqueue[index]->async) {
2152                 if (dev->virtqueue[index]->async->pkts_inflight_n) {
2153                         VHOST_LOG_CONFIG(ERR,
2154                                 "(%s) failed to enable vring. Inflight packets must be completed first\n",
2155                                 dev->ifname);
2156                         return RTE_VHOST_MSG_RESULT_ERR;
2157                 }
2158         }
2159
2160         dev->virtqueue[index]->enabled = enable;
2161
2162         return RTE_VHOST_MSG_RESULT_OK;
2163 }
2164
2165 static int
2166 vhost_user_get_protocol_features(struct virtio_net **pdev,
2167                         struct vhu_msg_context *ctx,
2168                         int main_fd __rte_unused)
2169 {
2170         struct virtio_net *dev = *pdev;
2171         uint64_t features, protocol_features;
2172
2173         rte_vhost_driver_get_features(dev->ifname, &features);
2174         rte_vhost_driver_get_protocol_features(dev->ifname, &protocol_features);
2175
2176         ctx->msg.payload.u64 = protocol_features;
2177         ctx->msg.size = sizeof(ctx->msg.payload.u64);
2178         ctx->fd_num = 0;
2179
2180         return RTE_VHOST_MSG_RESULT_REPLY;
2181 }
2182
2183 static int
2184 vhost_user_set_protocol_features(struct virtio_net **pdev,
2185                         struct vhu_msg_context *ctx,
2186                         int main_fd __rte_unused)
2187 {
2188         struct virtio_net *dev = *pdev;
2189         uint64_t protocol_features = ctx->msg.payload.u64;
2190         uint64_t slave_protocol_features = 0;
2191
2192         rte_vhost_driver_get_protocol_features(dev->ifname,
2193                         &slave_protocol_features);
2194         if (protocol_features & ~slave_protocol_features) {
2195                 VHOST_LOG_CONFIG(ERR, "(%s) received invalid protocol features.\n", dev->ifname);
2196                 return RTE_VHOST_MSG_RESULT_ERR;
2197         }
2198
2199         dev->protocol_features = protocol_features;
2200         VHOST_LOG_CONFIG(INFO, "(%s) negotiated Vhost-user protocol features: 0x%" PRIx64 "\n",
2201                 dev->ifname, dev->protocol_features);
2202
2203         return RTE_VHOST_MSG_RESULT_OK;
2204 }
2205
2206 static int
2207 vhost_user_set_log_base(struct virtio_net **pdev,
2208                         struct vhu_msg_context *ctx,
2209                         int main_fd __rte_unused)
2210 {
2211         struct virtio_net *dev = *pdev;
2212         int fd = ctx->fds[0];
2213         uint64_t size, off;
2214         void *addr;
2215         uint32_t i;
2216
2217         if (validate_msg_fds(dev, ctx, 1) != 0)
2218                 return RTE_VHOST_MSG_RESULT_ERR;
2219
2220         if (fd < 0) {
2221                 VHOST_LOG_CONFIG(ERR, "(%s) invalid log fd: %d\n", dev->ifname, fd);
2222                 return RTE_VHOST_MSG_RESULT_ERR;
2223         }
2224
2225         if (ctx->msg.size != sizeof(VhostUserLog)) {
2226                 VHOST_LOG_CONFIG(ERR, "(%s) invalid log base msg size: %"PRId32" != %d\n",
2227                         dev->ifname, ctx->msg.size, (int)sizeof(VhostUserLog));
2228                 goto close_msg_fds;
2229         }
2230
2231         size = ctx->msg.payload.log.mmap_size;
2232         off  = ctx->msg.payload.log.mmap_offset;
2233
2234         /* Check for mmap size and offset overflow. */
2235         if (off >= -size) {
2236                 VHOST_LOG_CONFIG(ERR,
2237                                 "(%s) log offset %#"PRIx64" and log size %#"PRIx64" overflow\n",
2238                                 dev->ifname, off, size);
2239                 goto close_msg_fds;
2240         }
2241
2242         VHOST_LOG_CONFIG(INFO, "(%s) log mmap size: %"PRId64", offset: %"PRId64"\n",
2243                         dev->ifname, size, off);
2244
2245         /*
2246          * mmap from 0 to workaround a hugepage mmap bug: mmap will
2247          * fail when offset is not page size aligned.
2248          */
2249         addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
2250         close(fd);
2251         if (addr == MAP_FAILED) {
2252                 VHOST_LOG_CONFIG(ERR, "(%s) mmap log base failed!\n", dev->ifname);
2253                 return RTE_VHOST_MSG_RESULT_ERR;
2254         }
2255
2256         /*
2257          * Free previously mapped log memory on occasionally
2258          * multiple VHOST_USER_SET_LOG_BASE.
2259          */
2260         if (dev->log_addr) {
2261                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
2262         }
2263         dev->log_addr = (uint64_t)(uintptr_t)addr;
2264         dev->log_base = dev->log_addr + off;
2265         dev->log_size = size;
2266
2267         for (i = 0; i < dev->nr_vring; i++) {
2268                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2269
2270                 rte_free(vq->log_cache);
2271                 vq->log_cache = NULL;
2272                 vq->log_cache_nb_elem = 0;
2273                 vq->log_cache = rte_malloc_socket("vq log cache",
2274                                 sizeof(struct log_cache_entry) * VHOST_LOG_CACHE_NR,
2275                                 0, vq->numa_node);
2276                 /*
2277                  * If log cache alloc fail, don't fail migration, but no
2278                  * caching will be done, which will impact performance
2279                  */
2280                 if (!vq->log_cache)
2281                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate VQ logging cache\n",
2282                                         dev->ifname);
2283         }
2284
2285         /*
2286          * The spec is not clear about it (yet), but QEMU doesn't expect
2287          * any payload in the reply.
2288          */
2289         ctx->msg.size = 0;
2290         ctx->fd_num = 0;
2291
2292         return RTE_VHOST_MSG_RESULT_REPLY;
2293
2294 close_msg_fds:
2295         close_msg_fds(ctx);
2296         return RTE_VHOST_MSG_RESULT_ERR;
2297 }
2298
2299 static int vhost_user_set_log_fd(struct virtio_net **pdev,
2300                         struct vhu_msg_context *ctx,
2301                         int main_fd __rte_unused)
2302 {
2303         struct virtio_net *dev = *pdev;
2304
2305         if (validate_msg_fds(dev, ctx, 1) != 0)
2306                 return RTE_VHOST_MSG_RESULT_ERR;
2307
2308         close(ctx->fds[0]);
2309         VHOST_LOG_CONFIG(INFO, "(%s) not implemented.\n", dev->ifname);
2310
2311         return RTE_VHOST_MSG_RESULT_OK;
2312 }
2313
2314 /*
2315  * An rarp packet is constructed and broadcasted to notify switches about
2316  * the new location of the migrated VM, so that packets from outside will
2317  * not be lost after migration.
2318  *
2319  * However, we don't actually "send" a rarp packet here, instead, we set
2320  * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
2321  */
2322 static int
2323 vhost_user_send_rarp(struct virtio_net **pdev,
2324                         struct vhu_msg_context *ctx,
2325                         int main_fd __rte_unused)
2326 {
2327         struct virtio_net *dev = *pdev;
2328         uint8_t *mac = (uint8_t *)&ctx->msg.payload.u64;
2329         struct rte_vdpa_device *vdpa_dev;
2330
2331         VHOST_LOG_CONFIG(DEBUG, "(%s) MAC: " RTE_ETHER_ADDR_PRT_FMT "\n",
2332                 dev->ifname, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
2333         memcpy(dev->mac.addr_bytes, mac, 6);
2334
2335         /*
2336          * Set the flag to inject a RARP broadcast packet at
2337          * rte_vhost_dequeue_burst().
2338          *
2339          * __ATOMIC_RELEASE ordering is for making sure the mac is
2340          * copied before the flag is set.
2341          */
2342         __atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
2343         vdpa_dev = dev->vdpa_dev;
2344         if (vdpa_dev && vdpa_dev->ops->migration_done)
2345                 vdpa_dev->ops->migration_done(dev->vid);
2346
2347         return RTE_VHOST_MSG_RESULT_OK;
2348 }
2349
2350 static int
2351 vhost_user_net_set_mtu(struct virtio_net **pdev,
2352                         struct vhu_msg_context *ctx,
2353                         int main_fd __rte_unused)
2354 {
2355         struct virtio_net *dev = *pdev;
2356
2357         if (ctx->msg.payload.u64 < VIRTIO_MIN_MTU ||
2358                         ctx->msg.payload.u64 > VIRTIO_MAX_MTU) {
2359                 VHOST_LOG_CONFIG(ERR, "(%s) invalid MTU size (%"PRIu64")\n",
2360                                 dev->ifname, ctx->msg.payload.u64);
2361
2362                 return RTE_VHOST_MSG_RESULT_ERR;
2363         }
2364
2365         dev->mtu = ctx->msg.payload.u64;
2366
2367         return RTE_VHOST_MSG_RESULT_OK;
2368 }
2369
2370 static int
2371 vhost_user_set_req_fd(struct virtio_net **pdev,
2372                         struct vhu_msg_context *ctx,
2373                         int main_fd __rte_unused)
2374 {
2375         struct virtio_net *dev = *pdev;
2376         int fd = ctx->fds[0];
2377
2378         if (validate_msg_fds(dev, ctx, 1) != 0)
2379                 return RTE_VHOST_MSG_RESULT_ERR;
2380
2381         if (fd < 0) {
2382                 VHOST_LOG_CONFIG(ERR, "(%s) invalid file descriptor for slave channel (%d)\n",
2383                                 dev->ifname, fd);
2384                 return RTE_VHOST_MSG_RESULT_ERR;
2385         }
2386
2387         if (dev->slave_req_fd >= 0)
2388                 close(dev->slave_req_fd);
2389
2390         dev->slave_req_fd = fd;
2391
2392         return RTE_VHOST_MSG_RESULT_OK;
2393 }
2394
2395 static int
2396 is_vring_iotlb_split(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2397 {
2398         struct vhost_vring_addr *ra;
2399         uint64_t start, end, len;
2400
2401         start = imsg->iova;
2402         end = start + imsg->size;
2403
2404         ra = &vq->ring_addrs;
2405         len = sizeof(struct vring_desc) * vq->size;
2406         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2407                 return 1;
2408
2409         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
2410         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2411                 return 1;
2412
2413         len = sizeof(struct vring_used) +
2414                sizeof(struct vring_used_elem) * vq->size;
2415         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2416                 return 1;
2417
2418         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2419                 len = sizeof(uint64_t);
2420                 if (ra->log_guest_addr < end &&
2421                     (ra->log_guest_addr + len) > start)
2422                         return 1;
2423         }
2424
2425         return 0;
2426 }
2427
2428 static int
2429 is_vring_iotlb_packed(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2430 {
2431         struct vhost_vring_addr *ra;
2432         uint64_t start, end, len;
2433
2434         start = imsg->iova;
2435         end = start + imsg->size;
2436
2437         ra = &vq->ring_addrs;
2438         len = sizeof(struct vring_packed_desc) * vq->size;
2439         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2440                 return 1;
2441
2442         len = sizeof(struct vring_packed_desc_event);
2443         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2444                 return 1;
2445
2446         len = sizeof(struct vring_packed_desc_event);
2447         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2448                 return 1;
2449
2450         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2451                 len = sizeof(uint64_t);
2452                 if (ra->log_guest_addr < end &&
2453                     (ra->log_guest_addr + len) > start)
2454                         return 1;
2455         }
2456
2457         return 0;
2458 }
2459
2460 static int is_vring_iotlb(struct virtio_net *dev,
2461                           struct vhost_virtqueue *vq,
2462                           struct vhost_iotlb_msg *imsg)
2463 {
2464         if (vq_is_packed(dev))
2465                 return is_vring_iotlb_packed(vq, imsg);
2466         else
2467                 return is_vring_iotlb_split(vq, imsg);
2468 }
2469
2470 static int
2471 vhost_user_get_config(struct virtio_net **pdev,
2472                         struct vhu_msg_context *ctx,
2473                         int main_fd __rte_unused)
2474 {
2475         struct virtio_net *dev = *pdev;
2476         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
2477         int ret = 0;
2478
2479         if (validate_msg_fds(dev, ctx, 0) != 0)
2480                 return RTE_VHOST_MSG_RESULT_ERR;
2481
2482         if (!vdpa_dev) {
2483                 VHOST_LOG_CONFIG(ERR, "(%s) is not vDPA device!\n",
2484                                  dev->ifname);
2485                 return RTE_VHOST_MSG_RESULT_ERR;
2486         }
2487
2488         if (vdpa_dev->ops->get_config) {
2489                 ret = vdpa_dev->ops->get_config(dev->vid,
2490                                            ctx->msg.payload.cfg.region,
2491                                            ctx->msg.payload.cfg.size);
2492                 if (ret != 0) {
2493                         ctx->msg.size = 0;
2494                         VHOST_LOG_CONFIG(ERR,
2495                                          "(%s) get_config() return error!\n",
2496                                          dev->ifname);
2497                 }
2498         } else {
2499                 VHOST_LOG_CONFIG(ERR, "(%s) get_config() not supported!\n",
2500                                  dev->ifname);
2501         }
2502
2503         return RTE_VHOST_MSG_RESULT_REPLY;
2504 }
2505
2506 static int
2507 vhost_user_set_config(struct virtio_net **pdev,
2508                         struct vhu_msg_context *ctx,
2509                         int main_fd __rte_unused)
2510 {
2511         struct virtio_net *dev = *pdev;
2512         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
2513         int ret = 0;
2514
2515         if (validate_msg_fds(dev, ctx, 0) != 0)
2516                 return RTE_VHOST_MSG_RESULT_ERR;
2517
2518         if (ctx->msg.payload.cfg.size > VHOST_USER_MAX_CONFIG_SIZE) {
2519                 VHOST_LOG_CONFIG(ERR,
2520                         "(%s) vhost_user_config size: %"PRIu32", should not be larger than %d\n",
2521                         dev->ifname, ctx->msg.payload.cfg.size,
2522                         VHOST_USER_MAX_CONFIG_SIZE);
2523                 goto out;
2524         }
2525
2526         if (!vdpa_dev) {
2527                 VHOST_LOG_CONFIG(ERR, "(%s) is not vDPA device!\n",
2528                                  dev->ifname);
2529                 goto out;
2530         }
2531
2532         if (vdpa_dev->ops->set_config) {
2533                 ret = vdpa_dev->ops->set_config(dev->vid,
2534                         ctx->msg.payload.cfg.region,
2535                         ctx->msg.payload.cfg.offset,
2536                         ctx->msg.payload.cfg.size,
2537                         ctx->msg.payload.cfg.flags);
2538                 if (ret)
2539                         VHOST_LOG_CONFIG(ERR,
2540                                          "(%s) set_config() return error!\n",
2541                                          dev->ifname);
2542         } else {
2543                 VHOST_LOG_CONFIG(ERR, "(%s) set_config() not supported!\n",
2544                                  dev->ifname);
2545         }
2546
2547         return RTE_VHOST_MSG_RESULT_OK;
2548
2549 out:
2550         return RTE_VHOST_MSG_RESULT_ERR;
2551 }
2552
2553 static int
2554 vhost_user_iotlb_msg(struct virtio_net **pdev,
2555                         struct vhu_msg_context *ctx,
2556                         int main_fd __rte_unused)
2557 {
2558         struct virtio_net *dev = *pdev;
2559         struct vhost_iotlb_msg *imsg = &ctx->msg.payload.iotlb;
2560         uint16_t i;
2561         uint64_t vva, len;
2562
2563         switch (imsg->type) {
2564         case VHOST_IOTLB_UPDATE:
2565                 len = imsg->size;
2566                 vva = qva_to_vva(dev, imsg->uaddr, &len);
2567                 if (!vva)
2568                         return RTE_VHOST_MSG_RESULT_ERR;
2569
2570                 for (i = 0; i < dev->nr_vring; i++) {
2571                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2572
2573                         if (!vq)
2574                                 continue;
2575
2576                         vhost_user_iotlb_cache_insert(dev, vq, imsg->iova, vva,
2577                                         len, imsg->perm);
2578
2579                         if (is_vring_iotlb(dev, vq, imsg)) {
2580                                 rte_spinlock_lock(&vq->access_lock);
2581                                 *pdev = dev = translate_ring_addresses(dev, i);
2582                                 rte_spinlock_unlock(&vq->access_lock);
2583                         }
2584                 }
2585                 break;
2586         case VHOST_IOTLB_INVALIDATE:
2587                 for (i = 0; i < dev->nr_vring; i++) {
2588                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2589
2590                         if (!vq)
2591                                 continue;
2592
2593                         vhost_user_iotlb_cache_remove(vq, imsg->iova,
2594                                         imsg->size);
2595
2596                         if (is_vring_iotlb(dev, vq, imsg)) {
2597                                 rte_spinlock_lock(&vq->access_lock);
2598                                 vring_invalidate(dev, vq);
2599                                 rte_spinlock_unlock(&vq->access_lock);
2600                         }
2601                 }
2602                 break;
2603         default:
2604                 VHOST_LOG_CONFIG(ERR, "(%s) invalid IOTLB message type (%d)\n",
2605                                 dev->ifname, imsg->type);
2606                 return RTE_VHOST_MSG_RESULT_ERR;
2607         }
2608
2609         return RTE_VHOST_MSG_RESULT_OK;
2610 }
2611
2612 static int
2613 vhost_user_set_postcopy_advise(struct virtio_net **pdev,
2614                         struct vhu_msg_context *ctx,
2615                         int main_fd __rte_unused)
2616 {
2617         struct virtio_net *dev = *pdev;
2618 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
2619         struct uffdio_api api_struct;
2620
2621         dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
2622
2623         if (dev->postcopy_ufd == -1) {
2624                 VHOST_LOG_CONFIG(ERR, "(%s) userfaultfd not available: %s\n",
2625                         dev->ifname, strerror(errno));
2626                 return RTE_VHOST_MSG_RESULT_ERR;
2627         }
2628         api_struct.api = UFFD_API;
2629         api_struct.features = 0;
2630         if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
2631                 VHOST_LOG_CONFIG(ERR, "(%s) UFFDIO_API ioctl failure: %s\n",
2632                         dev->ifname, strerror(errno));
2633                 close(dev->postcopy_ufd);
2634                 dev->postcopy_ufd = -1;
2635                 return RTE_VHOST_MSG_RESULT_ERR;
2636         }
2637         ctx->fds[0] = dev->postcopy_ufd;
2638         ctx->fd_num = 1;
2639
2640         return RTE_VHOST_MSG_RESULT_REPLY;
2641 #else
2642         dev->postcopy_ufd = -1;
2643         ctx->fd_num = 0;
2644
2645         return RTE_VHOST_MSG_RESULT_ERR;
2646 #endif
2647 }
2648
2649 static int
2650 vhost_user_set_postcopy_listen(struct virtio_net **pdev,
2651                         struct vhu_msg_context *ctx __rte_unused,
2652                         int main_fd __rte_unused)
2653 {
2654         struct virtio_net *dev = *pdev;
2655
2656         if (dev->mem && dev->mem->nregions) {
2657                 VHOST_LOG_CONFIG(ERR, "(%s) regions already registered at postcopy-listen\n",
2658                                 dev->ifname);
2659                 return RTE_VHOST_MSG_RESULT_ERR;
2660         }
2661         dev->postcopy_listening = 1;
2662
2663         return RTE_VHOST_MSG_RESULT_OK;
2664 }
2665
2666 static int
2667 vhost_user_postcopy_end(struct virtio_net **pdev,
2668                         struct vhu_msg_context *ctx,
2669                         int main_fd __rte_unused)
2670 {
2671         struct virtio_net *dev = *pdev;
2672
2673         dev->postcopy_listening = 0;
2674         if (dev->postcopy_ufd >= 0) {
2675                 close(dev->postcopy_ufd);
2676                 dev->postcopy_ufd = -1;
2677         }
2678
2679         ctx->msg.payload.u64 = 0;
2680         ctx->msg.size = sizeof(ctx->msg.payload.u64);
2681         ctx->fd_num = 0;
2682
2683         return RTE_VHOST_MSG_RESULT_REPLY;
2684 }
2685
2686 static int
2687 vhost_user_get_status(struct virtio_net **pdev,
2688                       struct vhu_msg_context *ctx,
2689                       int main_fd __rte_unused)
2690 {
2691         struct virtio_net *dev = *pdev;
2692
2693         ctx->msg.payload.u64 = dev->status;
2694         ctx->msg.size = sizeof(ctx->msg.payload.u64);
2695         ctx->fd_num = 0;
2696
2697         return RTE_VHOST_MSG_RESULT_REPLY;
2698 }
2699
2700 static int
2701 vhost_user_set_status(struct virtio_net **pdev,
2702                         struct vhu_msg_context *ctx,
2703                         int main_fd __rte_unused)
2704 {
2705         struct virtio_net *dev = *pdev;
2706
2707         /* As per Virtio specification, the device status is 8bits long */
2708         if (ctx->msg.payload.u64 > UINT8_MAX) {
2709                 VHOST_LOG_CONFIG(ERR, "(%s) invalid VHOST_USER_SET_STATUS payload 0x%" PRIx64 "\n",
2710                                 dev->ifname, ctx->msg.payload.u64);
2711                 return RTE_VHOST_MSG_RESULT_ERR;
2712         }
2713
2714         dev->status = ctx->msg.payload.u64;
2715
2716         if ((dev->status & VIRTIO_DEVICE_STATUS_FEATURES_OK) &&
2717             (dev->flags & VIRTIO_DEV_FEATURES_FAILED)) {
2718                 VHOST_LOG_CONFIG(ERR,
2719                                 "(%s) FEATURES_OK bit is set but feature negotiation failed\n",
2720                                 dev->ifname);
2721                 /*
2722                  * Clear the bit to let the driver know about the feature
2723                  * negotiation failure
2724                  */
2725                 dev->status &= ~VIRTIO_DEVICE_STATUS_FEATURES_OK;
2726         }
2727
2728         VHOST_LOG_CONFIG(INFO, "(%s) new device status(0x%08x):\n", dev->ifname,
2729                         dev->status);
2730         VHOST_LOG_CONFIG(INFO, "(%s)\t-RESET: %u\n", dev->ifname,
2731                         (dev->status == VIRTIO_DEVICE_STATUS_RESET));
2732         VHOST_LOG_CONFIG(INFO, "(%s)\t-ACKNOWLEDGE: %u\n", dev->ifname,
2733                         !!(dev->status & VIRTIO_DEVICE_STATUS_ACK));
2734         VHOST_LOG_CONFIG(INFO, "(%s)\t-DRIVER: %u\n", dev->ifname,
2735                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER));
2736         VHOST_LOG_CONFIG(INFO, "(%s)\t-FEATURES_OK: %u\n", dev->ifname,
2737                         !!(dev->status & VIRTIO_DEVICE_STATUS_FEATURES_OK));
2738         VHOST_LOG_CONFIG(INFO, "(%s)\t-DRIVER_OK: %u\n", dev->ifname,
2739                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK));
2740         VHOST_LOG_CONFIG(INFO, "(%s)\t-DEVICE_NEED_RESET: %u\n", dev->ifname,
2741                         !!(dev->status & VIRTIO_DEVICE_STATUS_DEV_NEED_RESET));
2742         VHOST_LOG_CONFIG(INFO, "(%s)\t-FAILED: %u\n", dev->ifname,
2743                         !!(dev->status & VIRTIO_DEVICE_STATUS_FAILED));
2744
2745         return RTE_VHOST_MSG_RESULT_OK;
2746 }
2747
2748 #define VHOST_MESSAGE_HANDLERS \
2749 VHOST_MESSAGE_HANDLER(VHOST_USER_NONE, NULL, false) \
2750 VHOST_MESSAGE_HANDLER(VHOST_USER_GET_FEATURES, vhost_user_get_features, false) \
2751 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_FEATURES, vhost_user_set_features, false) \
2752 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_OWNER, vhost_user_set_owner, false) \
2753 VHOST_MESSAGE_HANDLER(VHOST_USER_RESET_OWNER, vhost_user_reset_owner, false) \
2754 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_MEM_TABLE, vhost_user_set_mem_table, true) \
2755 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_LOG_BASE, vhost_user_set_log_base, true) \
2756 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_LOG_FD, vhost_user_set_log_fd, true) \
2757 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_VRING_NUM, vhost_user_set_vring_num, false) \
2758 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_VRING_ADDR, vhost_user_set_vring_addr, false) \
2759 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_VRING_BASE, vhost_user_set_vring_base, false) \
2760 VHOST_MESSAGE_HANDLER(VHOST_USER_GET_VRING_BASE, vhost_user_get_vring_base, false) \
2761 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_VRING_KICK, vhost_user_set_vring_kick, true) \
2762 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_VRING_CALL, vhost_user_set_vring_call, true) \
2763 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_VRING_ERR, vhost_user_set_vring_err, true) \
2764 VHOST_MESSAGE_HANDLER(VHOST_USER_GET_PROTOCOL_FEATURES, vhost_user_get_protocol_features, false) \
2765 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_PROTOCOL_FEATURES, vhost_user_set_protocol_features, false) \
2766 VHOST_MESSAGE_HANDLER(VHOST_USER_GET_QUEUE_NUM, vhost_user_get_queue_num, false) \
2767 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_VRING_ENABLE, vhost_user_set_vring_enable, false) \
2768 VHOST_MESSAGE_HANDLER(VHOST_USER_SEND_RARP, vhost_user_send_rarp, false) \
2769 VHOST_MESSAGE_HANDLER(VHOST_USER_NET_SET_MTU, vhost_user_net_set_mtu, false) \
2770 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_SLAVE_REQ_FD, vhost_user_set_req_fd, true) \
2771 VHOST_MESSAGE_HANDLER(VHOST_USER_IOTLB_MSG, vhost_user_iotlb_msg, false) \
2772 VHOST_MESSAGE_HANDLER(VHOST_USER_GET_CONFIG, vhost_user_get_config, false) \
2773 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_CONFIG, vhost_user_set_config, false) \
2774 VHOST_MESSAGE_HANDLER(VHOST_USER_POSTCOPY_ADVISE, vhost_user_set_postcopy_advise, false) \
2775 VHOST_MESSAGE_HANDLER(VHOST_USER_POSTCOPY_LISTEN, vhost_user_set_postcopy_listen, false) \
2776 VHOST_MESSAGE_HANDLER(VHOST_USER_POSTCOPY_END, vhost_user_postcopy_end, false) \
2777 VHOST_MESSAGE_HANDLER(VHOST_USER_GET_INFLIGHT_FD, vhost_user_get_inflight_fd, false) \
2778 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_INFLIGHT_FD, vhost_user_set_inflight_fd, true) \
2779 VHOST_MESSAGE_HANDLER(VHOST_USER_SET_STATUS, vhost_user_set_status, false) \
2780 VHOST_MESSAGE_HANDLER(VHOST_USER_GET_STATUS, vhost_user_get_status, false)
2781
2782 #define VHOST_MESSAGE_HANDLER(id, handler, accepts_fd) \
2783         [id] = { #id, handler, accepts_fd },
2784 static vhost_message_handler_t vhost_message_handlers[] = {
2785         VHOST_MESSAGE_HANDLERS
2786 };
2787 #undef VHOST_MESSAGE_HANDLER
2788
2789 /* return bytes# of read on success or negative val on failure. */
2790 static int
2791 read_vhost_message(struct virtio_net *dev, int sockfd, struct  vhu_msg_context *ctx)
2792 {
2793         int ret;
2794
2795         ret = read_fd_message(dev->ifname, sockfd, (char *)&ctx->msg, VHOST_USER_HDR_SIZE,
2796                 ctx->fds, VHOST_MEMORY_MAX_NREGIONS, &ctx->fd_num);
2797         if (ret <= 0) {
2798                 return ret;
2799         } else if (ret != VHOST_USER_HDR_SIZE) {
2800                 VHOST_LOG_CONFIG(ERR, "(%s) Unexpected header size read\n", dev->ifname);
2801                 close_msg_fds(ctx);
2802                 return -1;
2803         }
2804
2805         if (ctx->msg.size) {
2806                 if (ctx->msg.size > sizeof(ctx->msg.payload)) {
2807                         VHOST_LOG_CONFIG(ERR, "(%s) invalid msg size: %d\n",
2808                                         dev->ifname, ctx->msg.size);
2809                         return -1;
2810                 }
2811                 ret = read(sockfd, &ctx->msg.payload, ctx->msg.size);
2812                 if (ret <= 0)
2813                         return ret;
2814                 if (ret != (int)ctx->msg.size) {
2815                         VHOST_LOG_CONFIG(ERR, "(%s) read control message failed\n", dev->ifname);
2816                         return -1;
2817                 }
2818         }
2819
2820         return ret;
2821 }
2822
2823 static int
2824 send_vhost_message(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx)
2825 {
2826         if (!ctx)
2827                 return 0;
2828
2829         return send_fd_message(dev->ifname, sockfd, (char *)&ctx->msg,
2830                 VHOST_USER_HDR_SIZE + ctx->msg.size, ctx->fds, ctx->fd_num);
2831 }
2832
2833 static int
2834 send_vhost_reply(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx)
2835 {
2836         if (!ctx)
2837                 return 0;
2838
2839         ctx->msg.flags &= ~VHOST_USER_VERSION_MASK;
2840         ctx->msg.flags &= ~VHOST_USER_NEED_REPLY;
2841         ctx->msg.flags |= VHOST_USER_VERSION;
2842         ctx->msg.flags |= VHOST_USER_REPLY_MASK;
2843
2844         return send_vhost_message(dev, sockfd, ctx);
2845 }
2846
2847 static int
2848 send_vhost_slave_message(struct virtio_net *dev,
2849                 struct vhu_msg_context *ctx)
2850 {
2851         int ret;
2852
2853         if (ctx->msg.flags & VHOST_USER_NEED_REPLY)
2854                 rte_spinlock_lock(&dev->slave_req_lock);
2855
2856         ret = send_vhost_message(dev, dev->slave_req_fd, ctx);
2857         if (ret < 0 && (ctx->msg.flags & VHOST_USER_NEED_REPLY))
2858                 rte_spinlock_unlock(&dev->slave_req_lock);
2859
2860         return ret;
2861 }
2862
2863 /*
2864  * Allocate a queue pair if it hasn't been allocated yet
2865  */
2866 static int
2867 vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev,
2868                         struct vhu_msg_context *ctx)
2869 {
2870         uint32_t vring_idx;
2871
2872         switch (ctx->msg.request.master) {
2873         case VHOST_USER_SET_VRING_KICK:
2874         case VHOST_USER_SET_VRING_CALL:
2875         case VHOST_USER_SET_VRING_ERR:
2876                 vring_idx = ctx->msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
2877                 break;
2878         case VHOST_USER_SET_VRING_NUM:
2879         case VHOST_USER_SET_VRING_BASE:
2880         case VHOST_USER_GET_VRING_BASE:
2881         case VHOST_USER_SET_VRING_ENABLE:
2882                 vring_idx = ctx->msg.payload.state.index;
2883                 break;
2884         case VHOST_USER_SET_VRING_ADDR:
2885                 vring_idx = ctx->msg.payload.addr.index;
2886                 break;
2887         case VHOST_USER_SET_INFLIGHT_FD:
2888                 vring_idx = ctx->msg.payload.inflight.num_queues - 1;
2889                 break;
2890         default:
2891                 return 0;
2892         }
2893
2894         if (vring_idx >= VHOST_MAX_VRING) {
2895                 VHOST_LOG_CONFIG(ERR, "(%s) invalid vring index: %u\n", dev->ifname, vring_idx);
2896                 return -1;
2897         }
2898
2899         if (dev->virtqueue[vring_idx])
2900                 return 0;
2901
2902         return alloc_vring_queue(dev, vring_idx);
2903 }
2904
2905 static void
2906 vhost_user_lock_all_queue_pairs(struct virtio_net *dev)
2907 {
2908         unsigned int i = 0;
2909         unsigned int vq_num = 0;
2910
2911         while (vq_num < dev->nr_vring) {
2912                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2913
2914                 if (vq) {
2915                         rte_spinlock_lock(&vq->access_lock);
2916                         vq_num++;
2917                 }
2918                 i++;
2919         }
2920 }
2921
2922 static void
2923 vhost_user_unlock_all_queue_pairs(struct virtio_net *dev)
2924 {
2925         unsigned int i = 0;
2926         unsigned int vq_num = 0;
2927
2928         while (vq_num < dev->nr_vring) {
2929                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2930
2931                 if (vq) {
2932                         rte_spinlock_unlock(&vq->access_lock);
2933                         vq_num++;
2934                 }
2935                 i++;
2936         }
2937 }
2938
2939 int
2940 vhost_user_msg_handler(int vid, int fd)
2941 {
2942         struct virtio_net *dev;
2943         struct vhu_msg_context ctx;
2944         vhost_message_handler_t *msg_handler;
2945         struct rte_vdpa_device *vdpa_dev;
2946         int ret;
2947         int unlock_required = 0;
2948         bool handled;
2949         uint32_t request;
2950         uint32_t i;
2951
2952         dev = get_device(vid);
2953         if (dev == NULL)
2954                 return -1;
2955
2956         if (!dev->notify_ops) {
2957                 dev->notify_ops = vhost_driver_callback_get(dev->ifname);
2958                 if (!dev->notify_ops) {
2959                         VHOST_LOG_CONFIG(ERR, "(%s) failed to get callback ops for driver\n",
2960                                 dev->ifname);
2961                         return -1;
2962                 }
2963         }
2964
2965         ret = read_vhost_message(dev, fd, &ctx);
2966         if (ret <= 0) {
2967                 if (ret < 0)
2968                         VHOST_LOG_CONFIG(ERR, "(%s) vhost read message failed\n", dev->ifname);
2969                 else
2970                         VHOST_LOG_CONFIG(INFO, "(%s) vhost peer closed\n", dev->ifname);
2971
2972                 return -1;
2973         }
2974
2975         request = ctx.msg.request.master;
2976         if (request > VHOST_USER_NONE && request < RTE_DIM(vhost_message_handlers))
2977                 msg_handler = &vhost_message_handlers[request];
2978         else
2979                 msg_handler = NULL;
2980
2981         if (msg_handler != NULL && msg_handler->description != NULL) {
2982                 if (request != VHOST_USER_IOTLB_MSG)
2983                         VHOST_LOG_CONFIG(INFO, "(%s) read message %s\n",
2984                                 dev->ifname, msg_handler->description);
2985                 else
2986                         VHOST_LOG_CONFIG(DEBUG, "(%s) read message %s\n",
2987                                 dev->ifname, msg_handler->description);
2988         } else {
2989                 VHOST_LOG_CONFIG(DEBUG, "(%s) external request %d\n", dev->ifname, request);
2990         }
2991
2992         ret = vhost_user_check_and_alloc_queue_pair(dev, &ctx);
2993         if (ret < 0) {
2994                 VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc queue\n", dev->ifname);
2995                 return -1;
2996         }
2997
2998         /*
2999          * Note: we don't lock all queues on VHOST_USER_GET_VRING_BASE
3000          * and VHOST_USER_RESET_OWNER, since it is sent when virtio stops
3001          * and device is destroyed. destroy_device waits for queues to be
3002          * inactive, so it is safe. Otherwise taking the access_lock
3003          * would cause a dead lock.
3004          */
3005         switch (request) {
3006         case VHOST_USER_SET_FEATURES:
3007         case VHOST_USER_SET_PROTOCOL_FEATURES:
3008         case VHOST_USER_SET_OWNER:
3009         case VHOST_USER_SET_MEM_TABLE:
3010         case VHOST_USER_SET_LOG_BASE:
3011         case VHOST_USER_SET_LOG_FD:
3012         case VHOST_USER_SET_VRING_NUM:
3013         case VHOST_USER_SET_VRING_ADDR:
3014         case VHOST_USER_SET_VRING_BASE:
3015         case VHOST_USER_SET_VRING_KICK:
3016         case VHOST_USER_SET_VRING_CALL:
3017         case VHOST_USER_SET_VRING_ERR:
3018         case VHOST_USER_SET_VRING_ENABLE:
3019         case VHOST_USER_SEND_RARP:
3020         case VHOST_USER_NET_SET_MTU:
3021         case VHOST_USER_SET_SLAVE_REQ_FD:
3022                 if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
3023                         vhost_user_lock_all_queue_pairs(dev);
3024                         unlock_required = 1;
3025                 }
3026                 break;
3027         default:
3028                 break;
3029
3030         }
3031
3032         handled = false;
3033         if (dev->extern_ops.pre_msg_handle) {
3034                 RTE_BUILD_BUG_ON(offsetof(struct vhu_msg_context, msg) != 0);
3035                 ret = (*dev->extern_ops.pre_msg_handle)(dev->vid, &ctx);
3036                 switch (ret) {
3037                 case RTE_VHOST_MSG_RESULT_REPLY:
3038                         send_vhost_reply(dev, fd, &ctx);
3039                         /* Fall-through */
3040                 case RTE_VHOST_MSG_RESULT_ERR:
3041                 case RTE_VHOST_MSG_RESULT_OK:
3042                         handled = true;
3043                         goto skip_to_post_handle;
3044                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
3045                 default:
3046                         break;
3047                 }
3048         }
3049
3050         if (msg_handler == NULL || msg_handler->callback == NULL)
3051                 goto skip_to_post_handle;
3052
3053         if (!msg_handler->accepts_fd && validate_msg_fds(dev, &ctx, 0) != 0) {
3054                 ret = RTE_VHOST_MSG_RESULT_ERR;
3055         } else {
3056                 ret = msg_handler->callback(&dev, &ctx, fd);
3057         }
3058
3059         switch (ret) {
3060         case RTE_VHOST_MSG_RESULT_ERR:
3061                 VHOST_LOG_CONFIG(ERR, "(%s) processing %s failed.\n",
3062                         dev->ifname, msg_handler->description);
3063                 handled = true;
3064                 break;
3065         case RTE_VHOST_MSG_RESULT_OK:
3066                 VHOST_LOG_CONFIG(DEBUG, "(%s) processing %s succeeded.\n",
3067                         dev->ifname, msg_handler->description);
3068                 handled = true;
3069                 break;
3070         case RTE_VHOST_MSG_RESULT_REPLY:
3071                 VHOST_LOG_CONFIG(DEBUG, "(%s) processing %s succeeded and needs reply.\n",
3072                         dev->ifname, msg_handler->description);
3073                 send_vhost_reply(dev, fd, &ctx);
3074                 handled = true;
3075                 break;
3076         default:
3077                 break;
3078         }
3079
3080 skip_to_post_handle:
3081         if (ret != RTE_VHOST_MSG_RESULT_ERR &&
3082                         dev->extern_ops.post_msg_handle) {
3083                 RTE_BUILD_BUG_ON(offsetof(struct vhu_msg_context, msg) != 0);
3084                 ret = (*dev->extern_ops.post_msg_handle)(dev->vid, &ctx);
3085                 switch (ret) {
3086                 case RTE_VHOST_MSG_RESULT_REPLY:
3087                         send_vhost_reply(dev, fd, &ctx);
3088                         /* Fall-through */
3089                 case RTE_VHOST_MSG_RESULT_ERR:
3090                 case RTE_VHOST_MSG_RESULT_OK:
3091                         handled = true;
3092                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
3093                 default:
3094                         break;
3095                 }
3096         }
3097
3098         /* If message was not handled at this stage, treat it as an error */
3099         if (!handled) {
3100                 VHOST_LOG_CONFIG(ERR, "(%s) vhost message (req: %d) was not handled.\n",
3101                                 dev->ifname, request);
3102                 close_msg_fds(&ctx);
3103                 ret = RTE_VHOST_MSG_RESULT_ERR;
3104         }
3105
3106         /*
3107          * If the request required a reply that was already sent,
3108          * this optional reply-ack won't be sent as the
3109          * VHOST_USER_NEED_REPLY was cleared in send_vhost_reply().
3110          */
3111         if (ctx.msg.flags & VHOST_USER_NEED_REPLY) {
3112                 ctx.msg.payload.u64 = ret == RTE_VHOST_MSG_RESULT_ERR;
3113                 ctx.msg.size = sizeof(ctx.msg.payload.u64);
3114                 ctx.fd_num = 0;
3115                 send_vhost_reply(dev, fd, &ctx);
3116         } else if (ret == RTE_VHOST_MSG_RESULT_ERR) {
3117                 VHOST_LOG_CONFIG(ERR, "(%s) vhost message handling failed.\n", dev->ifname);
3118                 ret = -1;
3119                 goto unlock;
3120         }
3121
3122         ret = 0;
3123         for (i = 0; i < dev->nr_vring; i++) {
3124                 struct vhost_virtqueue *vq = dev->virtqueue[i];
3125                 bool cur_ready = vq_is_ready(dev, vq);
3126
3127                 if (cur_ready != (vq && vq->ready)) {
3128                         vq->ready = cur_ready;
3129                         vhost_user_notify_queue_state(dev, i, cur_ready);
3130                 }
3131         }
3132
3133 unlock:
3134         if (unlock_required)
3135                 vhost_user_unlock_all_queue_pairs(dev);
3136
3137         if (ret != 0 || !virtio_is_ready(dev))
3138                 goto out;
3139
3140         /*
3141          * Virtio is now ready. If not done already, it is time
3142          * to notify the application it can process the rings and
3143          * configure the vDPA device if present.
3144          */
3145
3146         if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
3147                 if (dev->notify_ops->new_device(dev->vid) == 0)
3148                         dev->flags |= VIRTIO_DEV_RUNNING;
3149         }
3150
3151         vdpa_dev = dev->vdpa_dev;
3152         if (!vdpa_dev)
3153                 goto out;
3154
3155         if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
3156                 if (vdpa_dev->ops->dev_conf(dev->vid))
3157                         VHOST_LOG_CONFIG(ERR, "(%s) failed to configure vDPA device\n",
3158                                         dev->ifname);
3159                 else
3160                         dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED;
3161         }
3162
3163 out:
3164         return ret;
3165 }
3166
3167 static int process_slave_message_reply(struct virtio_net *dev,
3168                                        const struct vhu_msg_context *ctx)
3169 {
3170         struct vhu_msg_context msg_reply;
3171         int ret;
3172
3173         if ((ctx->msg.flags & VHOST_USER_NEED_REPLY) == 0)
3174                 return 0;
3175
3176         ret = read_vhost_message(dev, dev->slave_req_fd, &msg_reply);
3177         if (ret <= 0) {
3178                 if (ret < 0)
3179                         VHOST_LOG_CONFIG(ERR, "(%s) vhost read slave message reply failed\n",
3180                                         dev->ifname);
3181                 else
3182                         VHOST_LOG_CONFIG(INFO, "(%s) vhost peer closed\n", dev->ifname);
3183                 ret = -1;
3184                 goto out;
3185         }
3186
3187         ret = 0;
3188         if (msg_reply.msg.request.slave != ctx->msg.request.slave) {
3189                 VHOST_LOG_CONFIG(ERR, "(%s) received unexpected msg type (%u), expected %u\n",
3190                                 dev->ifname, msg_reply.msg.request.slave, ctx->msg.request.slave);
3191                 ret = -1;
3192                 goto out;
3193         }
3194
3195         ret = msg_reply.msg.payload.u64 ? -1 : 0;
3196
3197 out:
3198         rte_spinlock_unlock(&dev->slave_req_lock);
3199         return ret;
3200 }
3201
3202 int
3203 vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
3204 {
3205         int ret;
3206         struct vhu_msg_context ctx = {
3207                 .msg = {
3208                         .request.slave = VHOST_USER_SLAVE_IOTLB_MSG,
3209                         .flags = VHOST_USER_VERSION,
3210                         .size = sizeof(ctx.msg.payload.iotlb),
3211                         .payload.iotlb = {
3212                                 .iova = iova,
3213                                 .perm = perm,
3214                                 .type = VHOST_IOTLB_MISS,
3215                         },
3216                 },
3217         };
3218
3219         ret = send_vhost_message(dev, dev->slave_req_fd, &ctx);
3220         if (ret < 0) {
3221                 VHOST_LOG_CONFIG(ERR, "(%s) failed to send IOTLB miss message (%d)\n",
3222                                 dev->ifname, ret);
3223                 return ret;
3224         }
3225
3226         return 0;
3227 }
3228
3229 static int
3230 vhost_user_slave_config_change(struct virtio_net *dev, bool need_reply)
3231 {
3232         int ret;
3233         struct vhu_msg_context ctx = {
3234                 .msg = {
3235                         .request.slave = VHOST_USER_SLAVE_CONFIG_CHANGE_MSG,
3236                         .flags = VHOST_USER_VERSION,
3237                         .size = 0,
3238                 }
3239         };
3240
3241         if (need_reply)
3242                 ctx.msg.flags |= VHOST_USER_NEED_REPLY;
3243
3244         ret = send_vhost_slave_message(dev, &ctx);
3245         if (ret < 0) {
3246                 VHOST_LOG_CONFIG(ERR, "(%s) failed to send config change (%d)\n",
3247                                 dev->ifname, ret);
3248                 return ret;
3249         }
3250
3251         return process_slave_message_reply(dev, &ctx);
3252 }
3253
3254 int
3255 rte_vhost_slave_config_change(int vid, bool need_reply)
3256 {
3257         struct virtio_net *dev;
3258
3259         dev = get_device(vid);
3260         if (!dev)
3261                 return -ENODEV;
3262
3263         return vhost_user_slave_config_change(dev, need_reply);
3264 }
3265
3266 static int vhost_user_slave_set_vring_host_notifier(struct virtio_net *dev,
3267                                                     int index, int fd,
3268                                                     uint64_t offset,
3269                                                     uint64_t size)
3270 {
3271         int ret;
3272         struct vhu_msg_context ctx = {
3273                 .msg = {
3274                         .request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
3275                         .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY,
3276                         .size = sizeof(ctx.msg.payload.area),
3277                         .payload.area = {
3278                                 .u64 = index & VHOST_USER_VRING_IDX_MASK,
3279                                 .size = size,
3280                                 .offset = offset,
3281                         },
3282                 },
3283         };
3284
3285         if (fd < 0)
3286                 ctx.msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
3287         else {
3288                 ctx.fds[0] = fd;
3289                 ctx.fd_num = 1;
3290         }
3291
3292         ret = send_vhost_slave_message(dev, &ctx);
3293         if (ret < 0) {
3294                 VHOST_LOG_CONFIG(ERR, "(%s) failed to set host notifier (%d)\n",
3295                                 dev->ifname, ret);
3296                 return ret;
3297         }
3298
3299         return process_slave_message_reply(dev, &ctx);
3300 }
3301
3302 int rte_vhost_host_notifier_ctrl(int vid, uint16_t qid, bool enable)
3303 {
3304         struct virtio_net *dev;
3305         struct rte_vdpa_device *vdpa_dev;
3306         int vfio_device_fd, ret = 0;
3307         uint64_t offset, size;
3308         unsigned int i, q_start, q_last;
3309
3310         dev = get_device(vid);
3311         if (!dev)
3312                 return -ENODEV;
3313
3314         vdpa_dev = dev->vdpa_dev;
3315         if (vdpa_dev == NULL)
3316                 return -ENODEV;
3317
3318         if (!(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
3319             !(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) ||
3320             !(dev->protocol_features &
3321                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) ||
3322             !(dev->protocol_features &
3323                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD)) ||
3324             !(dev->protocol_features &
3325                         (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER)))
3326                 return -ENOTSUP;
3327
3328         if (qid == RTE_VHOST_QUEUE_ALL) {
3329                 q_start = 0;
3330                 q_last = dev->nr_vring - 1;
3331         } else {
3332                 if (qid >= dev->nr_vring)
3333                         return -EINVAL;
3334                 q_start = qid;
3335                 q_last = qid;
3336         }
3337
3338         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_vfio_device_fd, -ENOTSUP);
3339         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_notify_area, -ENOTSUP);
3340
3341         vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid);
3342         if (vfio_device_fd < 0)
3343                 return -ENOTSUP;
3344
3345         if (enable) {
3346                 for (i = q_start; i <= q_last; i++) {
3347                         if (vdpa_dev->ops->get_notify_area(vid, i, &offset,
3348                                         &size) < 0) {
3349                                 ret = -ENOTSUP;
3350                                 goto disable;
3351                         }
3352
3353                         if (vhost_user_slave_set_vring_host_notifier(dev, i,
3354                                         vfio_device_fd, offset, size) < 0) {
3355                                 ret = -EFAULT;
3356                                 goto disable;
3357                         }
3358                 }
3359         } else {
3360 disable:
3361                 for (i = q_start; i <= q_last; i++) {
3362                         vhost_user_slave_set_vring_host_notifier(dev, i, -1,
3363                                         0, 0);
3364                 }
3365         }
3366
3367         return ret;
3368 }