lib/vhost/vhost_user.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 /* Security model
   6  * --------------
   7  * The vhost-user protocol connection is an external interface, so it must be
   8  * robust against invalid inputs.
   9  *
  10  * This is important because the vhost-user master is only one step removed
  11  * from the guest.  Malicious guests that have escaped will then launch further
  12  * attacks from the vhost-user master.
  13  *
  14  * Even in deployments where guests are trusted, a bug in the vhost-user master
  15  * can still cause invalid messages to be sent.  Such messages must not
  16  * compromise the stability of the DPDK application by causing crashes, memory
  17  * corruption, or other problematic behavior.
  18  *
  19  * Do not assume received VhostUserMsg fields contain sensible values!
  20  */
  21
  22 #include <stdint.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28 #include <sys/ioctl.h>
  29 #include <sys/mman.h>
  30 #include <sys/stat.h>
  31 #include <sys/syscall.h>
  32 #ifdef RTE_LIBRTE_VHOST_NUMA
  33 #include <numaif.h>
  34 #endif
  35 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
  36 #include <linux/userfaultfd.h>
  37 #endif
  38 #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
  39 #include <linux/memfd.h>
  40 #define MEMFD_SUPPORTED
  41 #endif
  42
  43 #include <rte_common.h>
  44 #include <rte_malloc.h>
  45 #include <rte_log.h>
  46 #include <rte_vfio.h>
  47 #include <rte_errno.h>
  48
  49 #include "iotlb.h"
  50 #include "vhost.h"
  51 #include "vhost_user.h"
  52
  53 #define VIRTIO_MIN_MTU 68
  54 #define VIRTIO_MAX_MTU 65535
  55
  56 #define INFLIGHT_ALIGNMENT      64
  57 #define INFLIGHT_VERSION        0x1
  58
  59 static const char *vhost_message_str[VHOST_USER_MAX] = {
  60         [VHOST_USER_NONE] = "VHOST_USER_NONE",
  61         [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
  62         [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
  63         [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
  64         [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
  65         [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
  66         [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
  67         [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
  68         [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
  69         [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
  70         [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
  71         [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
  72         [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
  73         [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
  74         [VHOST_USER_SET_VRING_ERR]  = "VHOST_USER_SET_VRING_ERR",
  75         [VHOST_USER_GET_PROTOCOL_FEATURES]  = "VHOST_USER_GET_PROTOCOL_FEATURES",
  76         [VHOST_USER_SET_PROTOCOL_FEATURES]  = "VHOST_USER_SET_PROTOCOL_FEATURES",
  77         [VHOST_USER_GET_QUEUE_NUM]  = "VHOST_USER_GET_QUEUE_NUM",
  78         [VHOST_USER_SET_VRING_ENABLE]  = "VHOST_USER_SET_VRING_ENABLE",
  79         [VHOST_USER_SEND_RARP]  = "VHOST_USER_SEND_RARP",
  80         [VHOST_USER_NET_SET_MTU]  = "VHOST_USER_NET_SET_MTU",
  81         [VHOST_USER_SET_SLAVE_REQ_FD]  = "VHOST_USER_SET_SLAVE_REQ_FD",
  82         [VHOST_USER_IOTLB_MSG]  = "VHOST_USER_IOTLB_MSG",
  83         [VHOST_USER_CRYPTO_CREATE_SESS] = "VHOST_USER_CRYPTO_CREATE_SESS",
  84         [VHOST_USER_CRYPTO_CLOSE_SESS] = "VHOST_USER_CRYPTO_CLOSE_SESS",
  85         [VHOST_USER_POSTCOPY_ADVISE]  = "VHOST_USER_POSTCOPY_ADVISE",
  86         [VHOST_USER_POSTCOPY_LISTEN]  = "VHOST_USER_POSTCOPY_LISTEN",
  87         [VHOST_USER_POSTCOPY_END]  = "VHOST_USER_POSTCOPY_END",
  88         [VHOST_USER_GET_INFLIGHT_FD] = "VHOST_USER_GET_INFLIGHT_FD",
  89         [VHOST_USER_SET_INFLIGHT_FD] = "VHOST_USER_SET_INFLIGHT_FD",
  90         [VHOST_USER_SET_STATUS] = "VHOST_USER_SET_STATUS",
  91         [VHOST_USER_GET_STATUS] = "VHOST_USER_GET_STATUS",
  92 };
  93
  94 static int send_vhost_reply(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx);
  95 static int read_vhost_message(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx);
  96
  97 static void
  98 close_msg_fds(struct vhu_msg_context *ctx)
  99 {
 100         int i;
 101
 102         for (i = 0; i < ctx->fd_num; i++) {
 103                 int fd = ctx->fds[i];
 104
 105                 if (fd == -1)
 106                         continue;
 107
 108                 ctx->fds[i] = -1;
 109                 close(fd);
 110         }
 111 }
 112
 113 /*
 114  * Ensure the expected number of FDs is received,
 115  * close all FDs and return an error if this is not the case.
 116  */
 117 static int
 118 validate_msg_fds(struct virtio_net *dev, struct vhu_msg_context *ctx, int expected_fds)
 119 {
 120         if (ctx->fd_num == expected_fds)
 121                 return 0;
 122
 123         VHOST_LOG_CONFIG(ERR, "(%s) expect %d FDs for request %s, received %d\n",
 124                 dev->ifname, expected_fds,
 125                 vhost_message_str[ctx->msg.request.master],
 126                 ctx->fd_num);
 127
 128         close_msg_fds(ctx);
 129
 130         return -1;
 131 }
 132
 133 static uint64_t
 134 get_blk_size(int fd)
 135 {
 136         struct stat stat;
 137         int ret;
 138
 139         ret = fstat(fd, &stat);
 140         return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
 141 }
 142
 143 static void
 144 async_dma_map(struct virtio_net *dev, bool do_map)
 145 {
 146         int ret = 0;
 147         uint32_t i;
 148         struct guest_page *page;
 149
 150         if (do_map) {
 151                 for (i = 0; i < dev->nr_guest_pages; i++) {
 152                         page = &dev->guest_pages[i];
 153                         ret = rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
 154                                                          page->host_user_addr,
 155                                                          page->host_iova,
 156                                                          page->size);
 157                         if (ret) {
 158                                 /*
 159                                  * DMA device may bind with kernel driver, in this case,
 160                                  * we don't need to program IOMMU manually. However, if no
 161                                  * device is bound with vfio/uio in DPDK, and vfio kernel
 162                                  * module is loaded, the API will still be called and return
 163                                  * with ENODEV.
 164                                  *
 165                                  * DPDK vfio only returns ENODEV in very similar situations
 166                                  * (vfio either unsupported, or supported but no devices found).
 167                                  * Either way, no mappings could be performed. We treat it as
 168                                  * normal case in async path. This is a workaround.
 169                                  */
 170                                 if (rte_errno == ENODEV)
 171                                         return;
 172
 173                                 /* DMA mapping errors won't stop VHOST_USER_SET_MEM_TABLE. */
 174                                 VHOST_LOG_CONFIG(ERR, "DMA engine map failed\n");
 175                         }
 176                 }
 177
 178         } else {
 179                 for (i = 0; i < dev->nr_guest_pages; i++) {
 180                         page = &dev->guest_pages[i];
 181                         ret = rte_vfio_container_dma_unmap(RTE_VFIO_DEFAULT_CONTAINER_FD,
 182                                                            page->host_user_addr,
 183                                                            page->host_iova,
 184                                                            page->size);
 185                         if (ret) {
 186                                 /* like DMA map, ignore the kernel driver case when unmap. */
 187                                 if (rte_errno == EINVAL)
 188                                         return;
 189
 190                                 VHOST_LOG_CONFIG(ERR, "DMA engine unmap failed\n");
 191                         }
 192                 }
 193         }
 194 }
 195
 196 static void
 197 free_mem_region(struct virtio_net *dev)
 198 {
 199         uint32_t i;
 200         struct rte_vhost_mem_region *reg;
 201
 202         if (!dev || !dev->mem)
 203                 return;
 204
 205         if (dev->async_copy && rte_vfio_is_enabled("vfio"))
 206                 async_dma_map(dev, false);
 207
 208         for (i = 0; i < dev->mem->nregions; i++) {
 209                 reg = &dev->mem->regions[i];
 210                 if (reg->host_user_addr) {
 211                         munmap(reg->mmap_addr, reg->mmap_size);
 212                         close(reg->fd);
 213                 }
 214         }
 215 }
 216
 217 void
 218 vhost_backend_cleanup(struct virtio_net *dev)
 219 {
 220         struct rte_vdpa_device *vdpa_dev;
 221
 222         vdpa_dev = dev->vdpa_dev;
 223         if (vdpa_dev && vdpa_dev->ops->dev_cleanup != NULL)
 224                 vdpa_dev->ops->dev_cleanup(dev->vid);
 225
 226         if (dev->mem) {
 227                 free_mem_region(dev);
 228                 rte_free(dev->mem);
 229                 dev->mem = NULL;
 230         }
 231
 232         rte_free(dev->guest_pages);
 233         dev->guest_pages = NULL;
 234
 235         if (dev->log_addr) {
 236                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 237                 dev->log_addr = 0;
 238         }
 239
 240         if (dev->inflight_info) {
 241                 if (dev->inflight_info->addr) {
 242                         munmap(dev->inflight_info->addr,
 243                                dev->inflight_info->size);
 244                         dev->inflight_info->addr = NULL;
 245                 }
 246
 247                 if (dev->inflight_info->fd >= 0) {
 248                         close(dev->inflight_info->fd);
 249                         dev->inflight_info->fd = -1;
 250                 }
 251
 252                 rte_free(dev->inflight_info);
 253                 dev->inflight_info = NULL;
 254         }
 255
 256         if (dev->slave_req_fd >= 0) {
 257                 close(dev->slave_req_fd);
 258                 dev->slave_req_fd = -1;
 259         }
 260
 261         if (dev->postcopy_ufd >= 0) {
 262                 close(dev->postcopy_ufd);
 263                 dev->postcopy_ufd = -1;
 264         }
 265
 266         dev->postcopy_listening = 0;
 267 }
 268
 269 static void
 270 vhost_user_notify_queue_state(struct virtio_net *dev, uint16_t index,
 271                               int enable)
 272 {
 273         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
 274         struct vhost_virtqueue *vq = dev->virtqueue[index];
 275
 276         /* Configure guest notifications on enable */
 277         if (enable && vq->notif_enable != VIRTIO_UNINITIALIZED_NOTIF)
 278                 vhost_enable_guest_notification(dev, vq, vq->notif_enable);
 279
 280         if (vdpa_dev && vdpa_dev->ops->set_vring_state)
 281                 vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
 282
 283         if (dev->notify_ops->vring_state_changed)
 284                 dev->notify_ops->vring_state_changed(dev->vid,
 285                                 index, enable);
 286 }
 287
 288 /*
 289  * This function just returns success at the moment unless
 290  * the device hasn't been initialised.
 291  */
 292 static int
 293 vhost_user_set_owner(struct virtio_net **pdev,
 294                         struct vhu_msg_context *ctx,
 295                         int main_fd __rte_unused)
 296 {
 297         struct virtio_net *dev = *pdev;
 298
 299         if (validate_msg_fds(dev, ctx, 0) != 0)
 300                 return RTE_VHOST_MSG_RESULT_ERR;
 301
 302         return RTE_VHOST_MSG_RESULT_OK;
 303 }
 304
 305 static int
 306 vhost_user_reset_owner(struct virtio_net **pdev,
 307                         struct vhu_msg_context *ctx,
 308                         int main_fd __rte_unused)
 309 {
 310         struct virtio_net *dev = *pdev;
 311
 312         if (validate_msg_fds(dev, ctx, 0) != 0)
 313                 return RTE_VHOST_MSG_RESULT_ERR;
 314
 315         vhost_destroy_device_notify(dev);
 316
 317         cleanup_device(dev, 0);
 318         reset_device(dev);
 319         return RTE_VHOST_MSG_RESULT_OK;
 320 }
 321
 322 /*
 323  * The features that we support are requested.
 324  */
 325 static int
 326 vhost_user_get_features(struct virtio_net **pdev,
 327                         struct vhu_msg_context *ctx,
 328                         int main_fd __rte_unused)
 329 {
 330         struct virtio_net *dev = *pdev;
 331         uint64_t features = 0;
 332
 333         if (validate_msg_fds(dev, ctx, 0) != 0)
 334                 return RTE_VHOST_MSG_RESULT_ERR;
 335
 336         rte_vhost_driver_get_features(dev->ifname, &features);
 337
 338         ctx->msg.payload.u64 = features;
 339         ctx->msg.size = sizeof(ctx->msg.payload.u64);
 340         ctx->fd_num = 0;
 341
 342         return RTE_VHOST_MSG_RESULT_REPLY;
 343 }
 344
 345 /*
 346  * The queue number that we support are requested.
 347  */
 348 static int
 349 vhost_user_get_queue_num(struct virtio_net **pdev,
 350                         struct vhu_msg_context *ctx,
 351                         int main_fd __rte_unused)
 352 {
 353         struct virtio_net *dev = *pdev;
 354         uint32_t queue_num = 0;
 355
 356         if (validate_msg_fds(dev, ctx, 0) != 0)
 357                 return RTE_VHOST_MSG_RESULT_ERR;
 358
 359         rte_vhost_driver_get_queue_num(dev->ifname, &queue_num);
 360
 361         ctx->msg.payload.u64 = (uint64_t)queue_num;
 362         ctx->msg.size = sizeof(ctx->msg.payload.u64);
 363         ctx->fd_num = 0;
 364
 365         return RTE_VHOST_MSG_RESULT_REPLY;
 366 }
 367
 368 /*
 369  * We receive the negotiated features supported by us and the virtio device.
 370  */
 371 static int
 372 vhost_user_set_features(struct virtio_net **pdev,
 373                         struct vhu_msg_context *ctx,
 374                         int main_fd __rte_unused)
 375 {
 376         struct virtio_net *dev = *pdev;
 377         uint64_t features = ctx->msg.payload.u64;
 378         uint64_t vhost_features = 0;
 379         struct rte_vdpa_device *vdpa_dev;
 380
 381         if (validate_msg_fds(dev, ctx, 0) != 0)
 382                 return RTE_VHOST_MSG_RESULT_ERR;
 383
 384         rte_vhost_driver_get_features(dev->ifname, &vhost_features);
 385         if (features & ~vhost_features) {
 386                 VHOST_LOG_CONFIG(ERR, "(%s) received invalid negotiated features.\n",
 387                         dev->ifname);
 388                 dev->flags |= VIRTIO_DEV_FEATURES_FAILED;
 389                 dev->status &= ~VIRTIO_DEVICE_STATUS_FEATURES_OK;
 390
 391                 return RTE_VHOST_MSG_RESULT_ERR;
 392         }
 393
 394         if (dev->flags & VIRTIO_DEV_RUNNING) {
 395                 if (dev->features == features)
 396                         return RTE_VHOST_MSG_RESULT_OK;
 397
 398                 /*
 399                  * Error out if master tries to change features while device is
 400                  * in running state. The exception being VHOST_F_LOG_ALL, which
 401                  * is enabled when the live-migration starts.
 402                  */
 403                 if ((dev->features ^ features) & ~(1ULL << VHOST_F_LOG_ALL)) {
 404                         VHOST_LOG_CONFIG(ERR, "(%s) features changed while device is running.\n",
 405                                 dev->ifname);
 406                         return RTE_VHOST_MSG_RESULT_ERR;
 407                 }
 408
 409                 if (dev->notify_ops->features_changed)
 410                         dev->notify_ops->features_changed(dev->vid, features);
 411         }
 412
 413         dev->features = features;
 414         if (dev->features &
 415                 ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
 416                  (1ULL << VIRTIO_F_VERSION_1) |
 417                  (1ULL << VIRTIO_F_RING_PACKED))) {
 418                 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 419         } else {
 420                 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
 421         }
 422         VHOST_LOG_CONFIG(INFO, "(%s) negotiated Virtio features: 0x%" PRIx64 "\n",
 423                         dev->ifname, dev->features);
 424         VHOST_LOG_CONFIG(DEBUG, "(%s) mergeable RX buffers %s, virtio 1 %s\n",
 425                 dev->ifname,
 426                 (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
 427                 (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
 428
 429         if ((dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) &&
 430             !(dev->features & (1ULL << VIRTIO_NET_F_MQ))) {
 431                 /*
 432                  * Remove all but first queue pair if MQ hasn't been
 433                  * negotiated. This is safe because the device is not
 434                  * running at this stage.
 435                  */
 436                 while (dev->nr_vring > 2) {
 437                         struct vhost_virtqueue *vq;
 438
 439                         vq = dev->virtqueue[--dev->nr_vring];
 440                         if (!vq)
 441                                 continue;
 442
 443                         dev->virtqueue[dev->nr_vring] = NULL;
 444                         cleanup_vq(vq, 1);
 445                         cleanup_vq_inflight(dev, vq);
 446                         free_vq(dev, vq);
 447                 }
 448         }
 449
 450         vdpa_dev = dev->vdpa_dev;
 451         if (vdpa_dev)
 452                 vdpa_dev->ops->set_features(dev->vid);
 453
 454         dev->flags &= ~VIRTIO_DEV_FEATURES_FAILED;
 455         return RTE_VHOST_MSG_RESULT_OK;
 456 }
 457
 458 /*
 459  * The virtio device sends us the size of the descriptor ring.
 460  */
 461 static int
 462 vhost_user_set_vring_num(struct virtio_net **pdev,
 463                         struct vhu_msg_context *ctx,
 464                         int main_fd __rte_unused)
 465 {
 466         struct virtio_net *dev = *pdev;
 467         struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index];
 468
 469         if (validate_msg_fds(dev, ctx, 0) != 0)
 470                 return RTE_VHOST_MSG_RESULT_ERR;
 471
 472         if (ctx->msg.payload.state.num > 32768) {
 473                 VHOST_LOG_CONFIG(ERR, "(%s) invalid virtqueue size %u\n",
 474                                 dev->ifname, ctx->msg.payload.state.num);
 475                 return RTE_VHOST_MSG_RESULT_ERR;
 476         }
 477
 478         vq->size = ctx->msg.payload.state.num;
 479
 480         /* VIRTIO 1.0, 2.4 Virtqueues says:
 481          *
 482          *   Queue Size value is always a power of 2. The maximum Queue Size
 483          *   value is 32768.
 484          *
 485          * VIRTIO 1.1 2.7 Virtqueues says:
 486          *
 487          *   Packed virtqueues support up to 2^15 entries each.
 488          */
 489         if (!vq_is_packed(dev)) {
 490                 if (vq->size & (vq->size - 1)) {
 491                         VHOST_LOG_CONFIG(ERR, "(%s) invalid virtqueue size %u\n",
 492                                         dev->ifname, vq->size);
 493                         return RTE_VHOST_MSG_RESULT_ERR;
 494                 }
 495         }
 496
 497         if (vq_is_packed(dev)) {
 498                 rte_free(vq->shadow_used_packed);
 499                 vq->shadow_used_packed = rte_malloc_socket(NULL,
 500                                 vq->size *
 501                                 sizeof(struct vring_used_elem_packed),
 502                                 RTE_CACHE_LINE_SIZE, vq->numa_node);
 503                 if (!vq->shadow_used_packed) {
 504                         VHOST_LOG_CONFIG(ERR,
 505                                 "(%s) failed to allocate memory for shadow used ring.\n",
 506                                 dev->ifname);
 507                         return RTE_VHOST_MSG_RESULT_ERR;
 508                 }
 509
 510         } else {
 511                 rte_free(vq->shadow_used_split);
 512
 513                 vq->shadow_used_split = rte_malloc_socket(NULL,
 514                                 vq->size * sizeof(struct vring_used_elem),
 515                                 RTE_CACHE_LINE_SIZE, vq->numa_node);
 516
 517                 if (!vq->shadow_used_split) {
 518                         VHOST_LOG_CONFIG(ERR,
 519                                 "(%s) failed to allocate memory for vq internal data.\n",
 520                                 dev->ifname);
 521                         return RTE_VHOST_MSG_RESULT_ERR;
 522                 }
 523         }
 524
 525         rte_free(vq->batch_copy_elems);
 526         vq->batch_copy_elems = rte_malloc_socket(NULL,
 527                                 vq->size * sizeof(struct batch_copy_elem),
 528                                 RTE_CACHE_LINE_SIZE, vq->numa_node);
 529         if (!vq->batch_copy_elems) {
 530                 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate memory for batching copy.\n",
 531                         dev->ifname);
 532                 return RTE_VHOST_MSG_RESULT_ERR;
 533         }
 534
 535         return RTE_VHOST_MSG_RESULT_OK;
 536 }
 537
 538 /*
 539  * Reallocate virtio_dev, vhost_virtqueue and related data structures to
 540  * make them on the same numa node as the memory of vring descriptor.
 541  */
 542 #ifdef RTE_LIBRTE_VHOST_NUMA
 543 static struct virtio_net*
 544 numa_realloc(struct virtio_net *dev, int index)
 545 {
 546         int node, dev_node;
 547         struct virtio_net *old_dev;
 548         struct vhost_virtqueue *vq;
 549         struct batch_copy_elem *bce;
 550         struct guest_page *gp;
 551         struct rte_vhost_memory *mem;
 552         size_t mem_size;
 553         int ret;
 554
 555         old_dev = dev;
 556         vq = dev->virtqueue[index];
 557
 558         /*
 559          * If VQ is ready, it is too late to reallocate, it certainly already
 560          * happened anyway on VHOST_USER_SET_VRING_ADRR.
 561          */
 562         if (vq->ready)
 563                 return dev;
 564
 565         ret = get_mempolicy(&node, NULL, 0, vq->desc, MPOL_F_NODE | MPOL_F_ADDR);
 566         if (ret) {
 567                 VHOST_LOG_CONFIG(ERR, "(%s) unable to get virtqueue %d numa information.\n",
 568                                 dev->ifname, index);
 569                 return dev;
 570         }
 571
 572         if (node == vq->numa_node)
 573                 goto out_dev_realloc;
 574
 575         vq = rte_realloc_socket(vq, sizeof(*vq), 0, node);
 576         if (!vq) {
 577                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc virtqueue %d on node %d\n",
 578                                 dev->ifname, index, node);
 579                 return dev;
 580         }
 581
 582         if (vq != dev->virtqueue[index]) {
 583                 VHOST_LOG_CONFIG(INFO, "(%s) reallocated virtqueue on node %d\n",
 584                                 dev->ifname, node);
 585                 dev->virtqueue[index] = vq;
 586                 vhost_user_iotlb_init(dev, index);
 587         }
 588
 589         if (vq_is_packed(dev)) {
 590                 struct vring_used_elem_packed *sup;
 591
 592                 sup = rte_realloc_socket(vq->shadow_used_packed, vq->size * sizeof(*sup),
 593                                 RTE_CACHE_LINE_SIZE, node);
 594                 if (!sup) {
 595                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc shadow packed on node %d\n",
 596                                         dev->ifname, node);
 597                         return dev;
 598                 }
 599                 vq->shadow_used_packed = sup;
 600         } else {
 601                 struct vring_used_elem *sus;
 602
 603                 sus = rte_realloc_socket(vq->shadow_used_split, vq->size * sizeof(*sus),
 604                                 RTE_CACHE_LINE_SIZE, node);
 605                 if (!sus) {
 606                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc shadow split on node %d\n",
 607                                         dev->ifname, node);
 608                         return dev;
 609                 }
 610                 vq->shadow_used_split = sus;
 611         }
 612
 613         bce = rte_realloc_socket(vq->batch_copy_elems, vq->size * sizeof(*bce),
 614                         RTE_CACHE_LINE_SIZE, node);
 615         if (!bce) {
 616                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc batch copy elem on node %d\n",
 617                                 dev->ifname, node);
 618                 return dev;
 619         }
 620         vq->batch_copy_elems = bce;
 621
 622         if (vq->log_cache) {
 623                 struct log_cache_entry *lc;
 624
 625                 lc = rte_realloc_socket(vq->log_cache, sizeof(*lc) * VHOST_LOG_CACHE_NR, 0, node);
 626                 if (!lc) {
 627                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc log cache on node %d\n",
 628                                         dev->ifname, node);
 629                         return dev;
 630                 }
 631                 vq->log_cache = lc;
 632         }
 633
 634         if (vq->resubmit_inflight) {
 635                 struct rte_vhost_resubmit_info *ri;
 636
 637                 ri = rte_realloc_socket(vq->resubmit_inflight, sizeof(*ri), 0, node);
 638                 if (!ri) {
 639                         VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc resubmit inflight on node %d\n",
 640                                         dev->ifname, node);
 641                         return dev;
 642                 }
 643                 vq->resubmit_inflight = ri;
 644
 645                 if (ri->resubmit_list) {
 646                         struct rte_vhost_resubmit_desc *rd;
 647
 648                         rd = rte_realloc_socket(ri->resubmit_list, sizeof(*rd) * ri->resubmit_num,
 649                                         0, node);
 650                         if (!rd) {
 651                                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc resubmit list on node %d\n",
 652                                                 dev->ifname, node);
 653                                 return dev;
 654                         }
 655                         ri->resubmit_list = rd;
 656                 }
 657         }
 658
 659         vq->numa_node = node;
 660
 661 out_dev_realloc:
 662
 663         if (dev->flags & VIRTIO_DEV_RUNNING)
 664                 return dev;
 665
 666         ret = get_mempolicy(&dev_node, NULL, 0, dev, MPOL_F_NODE | MPOL_F_ADDR);
 667         if (ret) {
 668                 VHOST_LOG_CONFIG(ERR, "(%s) unable to get numa information.\n", dev->ifname);
 669                 return dev;
 670         }
 671
 672         if (dev_node == node)
 673                 return dev;
 674
 675         dev = rte_realloc_socket(old_dev, sizeof(*dev), 0, node);
 676         if (!dev) {
 677                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc dev on node %d\n",
 678                                 old_dev->ifname, node);
 679                 return old_dev;
 680         }
 681
 682         VHOST_LOG_CONFIG(INFO, "(%s) reallocated device on node %d\n", dev->ifname, node);
 683         vhost_devices[dev->vid] = dev;
 684
 685         mem_size = sizeof(struct rte_vhost_memory) +
 686                 sizeof(struct rte_vhost_mem_region) * dev->mem->nregions;
 687         mem = rte_realloc_socket(dev->mem, mem_size, 0, node);
 688         if (!mem) {
 689                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc mem table on node %d\n",
 690                                 dev->ifname, node);
 691                 return dev;
 692         }
 693         dev->mem = mem;
 694
 695         gp = rte_realloc_socket(dev->guest_pages, dev->max_guest_pages * sizeof(*gp),
 696                         RTE_CACHE_LINE_SIZE, node);
 697         if (!gp) {
 698                 VHOST_LOG_CONFIG(ERR, "(%s) failed to realloc guest pages on node %d\n",
 699                                 dev->ifname, node);
 700                 return dev;
 701         }
 702         dev->guest_pages = gp;
 703
 704         return dev;
 705 }
 706 #else
 707 static struct virtio_net*
 708 numa_realloc(struct virtio_net *dev, int index __rte_unused)
 709 {
 710         return dev;
 711 }
 712 #endif
 713
 714 /* Converts QEMU virtual address to Vhost virtual address. */
 715 static uint64_t
 716 qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
 717 {
 718         struct rte_vhost_mem_region *r;
 719         uint32_t i;
 720
 721         if (unlikely(!dev || !dev->mem))
 722                 goto out_error;
 723
 724         /* Find the region where the address lives. */
 725         for (i = 0; i < dev->mem->nregions; i++) {
 726                 r = &dev->mem->regions[i];
 727
 728                 if (qva >= r->guest_user_addr &&
 729                     qva <  r->guest_user_addr + r->size) {
 730
 731                         if (unlikely(*len > r->guest_user_addr + r->size - qva))
 732                                 *len = r->guest_user_addr + r->size - qva;
 733
 734                         return qva - r->guest_user_addr +
 735                                r->host_user_addr;
 736                 }
 737         }
 738 out_error:
 739         *len = 0;
 740
 741         return 0;
 742 }
 743
 744
 745 /*
 746  * Converts ring address to Vhost virtual address.
 747  * If IOMMU is enabled, the ring address is a guest IO virtual address,
 748  * else it is a QEMU virtual address.
 749  */
 750 static uint64_t
 751 ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 752                 uint64_t ra, uint64_t *size)
 753 {
 754         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 755                 uint64_t vva;
 756
 757                 vhost_user_iotlb_rd_lock(vq);
 758                 vva = vhost_iova_to_vva(dev, vq, ra,
 759                                         size, VHOST_ACCESS_RW);
 760                 vhost_user_iotlb_rd_unlock(vq);
 761
 762                 return vva;
 763         }
 764
 765         return qva_to_vva(dev, ra, size);
 766 }
 767
 768 static uint64_t
 769 log_addr_to_gpa(struct virtio_net *dev, struct vhost_virtqueue *vq)
 770 {
 771         uint64_t log_gpa;
 772
 773         vhost_user_iotlb_rd_lock(vq);
 774         log_gpa = translate_log_addr(dev, vq, vq->ring_addrs.log_guest_addr);
 775         vhost_user_iotlb_rd_unlock(vq);
 776
 777         return log_gpa;
 778 }
 779
 780 static struct virtio_net *
 781 translate_ring_addresses(struct virtio_net *dev, int vq_index)
 782 {
 783         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 784         struct vhost_vring_addr *addr = &vq->ring_addrs;
 785         uint64_t len, expected_len;
 786
 787         if (addr->flags & (1 << VHOST_VRING_F_LOG)) {
 788                 vq->log_guest_addr =
 789                         log_addr_to_gpa(dev, vq);
 790                 if (vq->log_guest_addr == 0) {
 791                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map log_guest_addr.\n",
 792                                 dev->ifname);
 793                         return dev;
 794                 }
 795         }
 796
 797         if (vq_is_packed(dev)) {
 798                 len = sizeof(struct vring_packed_desc) * vq->size;
 799                 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 800                         ring_addr_to_vva(dev, vq, addr->desc_user_addr, &len);
 801                 if (vq->desc_packed == NULL ||
 802                                 len != sizeof(struct vring_packed_desc) *
 803                                 vq->size) {
 804                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map desc_packed ring.\n",
 805                                 dev->ifname);
 806                         return dev;
 807                 }
 808
 809                 dev = numa_realloc(dev, vq_index);
 810                 vq = dev->virtqueue[vq_index];
 811                 addr = &vq->ring_addrs;
 812
 813                 len = sizeof(struct vring_packed_desc_event);
 814                 vq->driver_event = (struct vring_packed_desc_event *)
 815                                         (uintptr_t)ring_addr_to_vva(dev,
 816                                         vq, addr->avail_user_addr, &len);
 817                 if (vq->driver_event == NULL ||
 818                                 len != sizeof(struct vring_packed_desc_event)) {
 819                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to find driver area address.\n",
 820                                 dev->ifname);
 821                         return dev;
 822                 }
 823
 824                 len = sizeof(struct vring_packed_desc_event);
 825                 vq->device_event = (struct vring_packed_desc_event *)
 826                                         (uintptr_t)ring_addr_to_vva(dev,
 827                                         vq, addr->used_user_addr, &len);
 828                 if (vq->device_event == NULL ||
 829                                 len != sizeof(struct vring_packed_desc_event)) {
 830                         VHOST_LOG_CONFIG(DEBUG, "(%s) failed to find device area address.\n",
 831                                 dev->ifname);
 832                         return dev;
 833                 }
 834
 835                 vq->access_ok = true;
 836                 return dev;
 837         }
 838
 839         /* The addresses are converted from QEMU virtual to Vhost virtual. */
 840         if (vq->desc && vq->avail && vq->used)
 841                 return dev;
 842
 843         len = sizeof(struct vring_desc) * vq->size;
 844         vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev,
 845                         vq, addr->desc_user_addr, &len);
 846         if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) {
 847                 VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map desc ring.\n", dev->ifname);
 848                 return dev;
 849         }
 850
 851         dev = numa_realloc(dev, vq_index);
 852         vq = dev->virtqueue[vq_index];
 853         addr = &vq->ring_addrs;
 854
 855         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
 856         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 857                 len += sizeof(uint16_t);
 858         expected_len = len;
 859         vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev,
 860                         vq, addr->avail_user_addr, &len);
 861         if (vq->avail == 0 || len != expected_len) {
 862                 VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map avail ring.\n", dev->ifname);
 863                 return dev;
 864         }
 865
 866         len = sizeof(struct vring_used) +
 867                 sizeof(struct vring_used_elem) * vq->size;
 868         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 869                 len += sizeof(uint16_t);
 870         expected_len = len;
 871         vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev,
 872                         vq, addr->used_user_addr, &len);
 873         if (vq->used == 0 || len != expected_len) {
 874                 VHOST_LOG_CONFIG(DEBUG, "(%s) failed to map used ring.\n", dev->ifname);
 875                 return dev;
 876         }
 877
 878         if (vq->last_used_idx != vq->used->idx) {
 879                 VHOST_LOG_CONFIG(WARNING, "(%s) last_used_idx (%u) and vq->used->idx (%u) mismatches;\n",
 880                         dev->ifname,
 881                         vq->last_used_idx, vq->used->idx);
 882                 vq->last_used_idx  = vq->used->idx;
 883                 vq->last_avail_idx = vq->used->idx;
 884                 VHOST_LOG_CONFIG(WARNING, "(%s) some packets maybe resent for Tx and dropped for Rx\n",
 885                         dev->ifname);
 886         }
 887
 888         vq->access_ok = true;
 889
 890         VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address desc: %p\n", dev->ifname, vq->desc);
 891         VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address avail: %p\n", dev->ifname, vq->avail);
 892         VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address used: %p\n", dev->ifname, vq->used);
 893         VHOST_LOG_CONFIG(DEBUG, "(%s) log_guest_addr: %" PRIx64 "\n",
 894                         dev->ifname, vq->log_guest_addr);
 895
 896         return dev;
 897 }
 898
 899 /*
 900  * The virtio device sends us the desc, used and avail ring addresses.
 901  * This function then converts these to our address space.
 902  */
 903 static int
 904 vhost_user_set_vring_addr(struct virtio_net **pdev,
 905                         struct vhu_msg_context *ctx,
 906                         int main_fd __rte_unused)
 907 {
 908         struct virtio_net *dev = *pdev;
 909         struct vhost_virtqueue *vq;
 910         struct vhost_vring_addr *addr = &ctx->msg.payload.addr;
 911         bool access_ok;
 912
 913         if (validate_msg_fds(dev, ctx, 0) != 0)
 914                 return RTE_VHOST_MSG_RESULT_ERR;
 915
 916         if (dev->mem == NULL)
 917                 return RTE_VHOST_MSG_RESULT_ERR;
 918
 919         /* addr->index refers to the queue index. The txq 1, rxq is 0. */
 920         vq = dev->virtqueue[ctx->msg.payload.addr.index];
 921
 922         access_ok = vq->access_ok;
 923
 924         /*
 925          * Rings addresses should not be interpreted as long as the ring is not
 926          * started and enabled
 927          */
 928         memcpy(&vq->ring_addrs, addr, sizeof(*addr));
 929
 930         vring_invalidate(dev, vq);
 931
 932         if ((vq->enabled && (dev->features &
 933                                 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) ||
 934                         access_ok) {
 935                 dev = translate_ring_addresses(dev, ctx->msg.payload.addr.index);
 936                 if (!dev)
 937                         return RTE_VHOST_MSG_RESULT_ERR;
 938
 939                 *pdev = dev;
 940         }
 941
 942         return RTE_VHOST_MSG_RESULT_OK;
 943 }
 944
 945 /*
 946  * The virtio device sends us the available ring last used index.
 947  */
 948 static int
 949 vhost_user_set_vring_base(struct virtio_net **pdev,
 950                         struct vhu_msg_context *ctx,
 951                         int main_fd __rte_unused)
 952 {
 953         struct virtio_net *dev = *pdev;
 954         struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index];
 955         uint64_t val = ctx->msg.payload.state.num;
 956
 957         if (validate_msg_fds(dev, ctx, 0) != 0)
 958                 return RTE_VHOST_MSG_RESULT_ERR;
 959
 960         if (vq_is_packed(dev)) {
 961                 /*
 962                  * Bit[0:14]: avail index
 963                  * Bit[15]: avail wrap counter
 964                  */
 965                 vq->last_avail_idx = val & 0x7fff;
 966                 vq->avail_wrap_counter = !!(val & (0x1 << 15));
 967                 /*
 968                  * Set used index to same value as available one, as
 969                  * their values should be the same since ring processing
 970                  * was stopped at get time.
 971                  */
 972                 vq->last_used_idx = vq->last_avail_idx;
 973                 vq->used_wrap_counter = vq->avail_wrap_counter;
 974         } else {
 975                 vq->last_used_idx = ctx->msg.payload.state.num;
 976                 vq->last_avail_idx = ctx->msg.payload.state.num;
 977         }
 978
 979         VHOST_LOG_CONFIG(INFO,
 980                 "(%s) vring base idx:%u last_used_idx:%u last_avail_idx:%u.\n",
 981                 dev->ifname, ctx->msg.payload.state.index, vq->last_used_idx,
 982                 vq->last_avail_idx);
 983
 984         return RTE_VHOST_MSG_RESULT_OK;
 985 }
 986
 987 static int
 988 add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
 989                    uint64_t host_iova, uint64_t host_user_addr, uint64_t size)
 990 {
 991         struct guest_page *page, *last_page;
 992         struct guest_page *old_pages;
 993
 994         if (dev->nr_guest_pages == dev->max_guest_pages) {
 995                 dev->max_guest_pages *= 2;
 996                 old_pages = dev->guest_pages;
 997                 dev->guest_pages = rte_realloc(dev->guest_pages,
 998                                         dev->max_guest_pages * sizeof(*page),
 999                                         RTE_CACHE_LINE_SIZE);
1000                 if (dev->guest_pages == NULL) {
1001                         VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
1002                         rte_free(old_pages);
1003                         return -1;
1004                 }
1005         }
1006
1007         if (dev->nr_guest_pages > 0) {
1008                 last_page = &dev->guest_pages[dev->nr_guest_pages - 1];
1009                 /* merge if the two pages are continuous */
1010                 if (host_iova == last_page->host_iova + last_page->size &&
1011                     guest_phys_addr == last_page->guest_phys_addr + last_page->size &&
1012                     host_user_addr == last_page->host_user_addr + last_page->size) {
1013                         last_page->size += size;
1014                         return 0;
1015                 }
1016         }
1017
1018         page = &dev->guest_pages[dev->nr_guest_pages++];
1019         page->guest_phys_addr = guest_phys_addr;
1020         page->host_iova  = host_iova;
1021         page->host_user_addr = host_user_addr;
1022         page->size = size;
1023
1024         return 0;
1025 }
1026
1027 static int
1028 add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
1029                 uint64_t page_size)
1030 {
1031         uint64_t reg_size = reg->size;
1032         uint64_t host_user_addr  = reg->host_user_addr;
1033         uint64_t guest_phys_addr = reg->guest_phys_addr;
1034         uint64_t host_iova;
1035         uint64_t size;
1036
1037         host_iova = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr);
1038         size = page_size - (guest_phys_addr & (page_size - 1));
1039         size = RTE_MIN(size, reg_size);
1040
1041         if (add_one_guest_page(dev, guest_phys_addr, host_iova,
1042                                host_user_addr, size) < 0)
1043                 return -1;
1044
1045         host_user_addr  += size;
1046         guest_phys_addr += size;
1047         reg_size -= size;
1048
1049         while (reg_size > 0) {
1050                 size = RTE_MIN(reg_size, page_size);
1051                 host_iova = rte_mem_virt2iova((void *)(uintptr_t)
1052                                                   host_user_addr);
1053                 if (add_one_guest_page(dev, guest_phys_addr, host_iova,
1054                                        host_user_addr, size) < 0)
1055                         return -1;
1056
1057                 host_user_addr  += size;
1058                 guest_phys_addr += size;
1059                 reg_size -= size;
1060         }
1061
1062         /* sort guest page array if over binary search threshold */
1063         if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
1064                 qsort((void *)dev->guest_pages, dev->nr_guest_pages,
1065                         sizeof(struct guest_page), guest_page_addrcmp);
1066         }
1067
1068         return 0;
1069 }
1070
1071 #ifdef RTE_LIBRTE_VHOST_DEBUG
1072 /* TODO: enable it only in debug mode? */
1073 static void
1074 dump_guest_pages(struct virtio_net *dev)
1075 {
1076         uint32_t i;
1077         struct guest_page *page;
1078
1079         for (i = 0; i < dev->nr_guest_pages; i++) {
1080                 page = &dev->guest_pages[i];
1081
1082                 VHOST_LOG_CONFIG(INFO, "(%s) guest physical page region %u\n",
1083                                 dev->ifname, i);
1084                 VHOST_LOG_CONFIG(INFO, "(%s)\tguest_phys_addr: %" PRIx64 "\n",
1085                                 dev->ifname, page->guest_phys_addr);
1086                 VHOST_LOG_CONFIG(INFO, "(%s)\thost_iova : %" PRIx64 "\n",
1087                                 dev->ifname, page->host_iova);
1088                 VHOST_LOG_CONFIG(INFO, "(%s)\tsize           : %" PRIx64 "\n",
1089                                 dev->ifname, page->size);
1090         }
1091 }
1092 #else
1093 #define dump_guest_pages(dev)
1094 #endif
1095
1096 static bool
1097 vhost_memory_changed(struct VhostUserMemory *new,
1098                      struct rte_vhost_memory *old)
1099 {
1100         uint32_t i;
1101
1102         if (new->nregions != old->nregions)
1103                 return true;
1104
1105         for (i = 0; i < new->nregions; ++i) {
1106                 VhostUserMemoryRegion *new_r = &new->regions[i];
1107                 struct rte_vhost_mem_region *old_r = &old->regions[i];
1108
1109                 if (new_r->guest_phys_addr != old_r->guest_phys_addr)
1110                         return true;
1111                 if (new_r->memory_size != old_r->size)
1112                         return true;
1113                 if (new_r->userspace_addr != old_r->guest_user_addr)
1114                         return true;
1115         }
1116
1117         return false;
1118 }
1119
1120 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
1121 static int
1122 vhost_user_postcopy_region_register(struct virtio_net *dev,
1123                 struct rte_vhost_mem_region *reg)
1124 {
1125         struct uffdio_register reg_struct;
1126
1127         /*
1128          * Let's register all the mmapped area to ensure
1129          * alignment on page boundary.
1130          */
1131         reg_struct.range.start = (uint64_t)(uintptr_t)reg->mmap_addr;
1132         reg_struct.range.len = reg->mmap_size;
1133         reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
1134
1135         if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER,
1136                                 &reg_struct)) {
1137                 VHOST_LOG_CONFIG(ERR, "(%s) failed to register ufd for region "
1138                                 "%" PRIx64 " - %" PRIx64 " (ufd = %d) %s\n",
1139                                 dev->ifname,
1140                                 (uint64_t)reg_struct.range.start,
1141                                 (uint64_t)reg_struct.range.start +
1142                                 (uint64_t)reg_struct.range.len - 1,
1143                                 dev->postcopy_ufd,
1144                                 strerror(errno));
1145                 return -1;
1146         }
1147
1148         VHOST_LOG_CONFIG(INFO,
1149                         "(%s)\t userfaultfd registered for range : %" PRIx64 " - %" PRIx64 "\n",
1150                         dev->ifname,
1151                         (uint64_t)reg_struct.range.start,
1152                         (uint64_t)reg_struct.range.start +
1153                         (uint64_t)reg_struct.range.len - 1);
1154
1155         return 0;
1156 }
1157 #else
1158 static int
1159 vhost_user_postcopy_region_register(struct virtio_net *dev __rte_unused,
1160                 struct rte_vhost_mem_region *reg __rte_unused)
1161 {
1162         return -1;
1163 }
1164 #endif
1165
1166 static int
1167 vhost_user_postcopy_register(struct virtio_net *dev, int main_fd,
1168                 struct vhu_msg_context *ctx)
1169 {
1170         struct VhostUserMemory *memory;
1171         struct rte_vhost_mem_region *reg;
1172         struct vhu_msg_context ack_ctx;
1173         uint32_t i;
1174
1175         if (!dev->postcopy_listening)
1176                 return 0;
1177
1178         /*
1179          * We haven't a better way right now than sharing
1180          * DPDK's virtual address with Qemu, so that Qemu can
1181          * retrieve the region offset when handling userfaults.
1182          */
1183         memory = &ctx->msg.payload.memory;
1184         for (i = 0; i < memory->nregions; i++) {
1185                 reg = &dev->mem->regions[i];
1186                 memory->regions[i].userspace_addr = reg->host_user_addr;
1187         }
1188
1189         /* Send the addresses back to qemu */
1190         ctx->fd_num = 0;
1191         send_vhost_reply(dev, main_fd, ctx);
1192
1193         /* Wait for qemu to acknowledge it got the addresses
1194          * we've got to wait before we're allowed to generate faults.
1195          */
1196         if (read_vhost_message(dev, main_fd, &ack_ctx) <= 0) {
1197                 VHOST_LOG_CONFIG(ERR, "(%s) failed to read qemu ack on postcopy set-mem-table\n",
1198                                 dev->ifname);
1199                 return -1;
1200         }
1201
1202         if (validate_msg_fds(dev, &ack_ctx, 0) != 0)
1203                 return -1;
1204
1205         if (ack_ctx.msg.request.master != VHOST_USER_SET_MEM_TABLE) {
1206                 VHOST_LOG_CONFIG(ERR, "(%s) bad qemu ack on postcopy set-mem-table (%d)\n",
1207                                 dev->ifname, ack_ctx.msg.request.master);
1208                 return -1;
1209         }
1210
1211         /* Now userfault register and we can use the memory */
1212         for (i = 0; i < memory->nregions; i++) {
1213                 reg = &dev->mem->regions[i];
1214                 if (vhost_user_postcopy_region_register(dev, reg) < 0)
1215                         return -1;
1216         }
1217
1218         return 0;
1219 }
1220
1221 static int
1222 vhost_user_mmap_region(struct virtio_net *dev,
1223                 struct rte_vhost_mem_region *region,
1224                 uint64_t mmap_offset)
1225 {
1226         void *mmap_addr;
1227         uint64_t mmap_size;
1228         uint64_t alignment;
1229         int populate;
1230
1231         /* Check for memory_size + mmap_offset overflow */
1232         if (mmap_offset >= -region->size) {
1233                 VHOST_LOG_CONFIG(ERR, "(%s) mmap_offset (%#"PRIx64") and memory_size (%#"PRIx64") overflow\n",
1234                                 dev->ifname, mmap_offset, region->size);
1235                 return -1;
1236         }
1237
1238         mmap_size = region->size + mmap_offset;
1239
1240         /* mmap() without flag of MAP_ANONYMOUS, should be called with length
1241          * argument aligned with hugepagesz at older longterm version Linux,
1242          * like 2.6.32 and 3.2.72, or mmap() will fail with EINVAL.
1243          *
1244          * To avoid failure, make sure in caller to keep length aligned.
1245          */
1246         alignment = get_blk_size(region->fd);
1247         if (alignment == (uint64_t)-1) {
1248                 VHOST_LOG_CONFIG(ERR, "(%s) couldn't get hugepage size through fstat\n",
1249                                 dev->ifname);
1250                 return -1;
1251         }
1252         mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
1253         if (mmap_size == 0) {
1254                 /*
1255                  * It could happen if initial mmap_size + alignment overflows
1256                  * the sizeof uint64, which could happen if either mmap_size or
1257                  * alignment value is wrong.
1258                  *
1259                  * mmap() kernel implementation would return an error, but
1260                  * better catch it before and provide useful info in the logs.
1261                  */
1262                 VHOST_LOG_CONFIG(ERR, "(%s) mmap size (0x%" PRIx64 ") or alignment (0x%" PRIx64 ") is invalid\n",
1263                                 dev->ifname, region->size + mmap_offset, alignment);
1264                 return -1;
1265         }
1266
1267         populate = dev->async_copy ? MAP_POPULATE : 0;
1268         mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
1269                         MAP_SHARED | populate, region->fd, 0);
1270
1271         if (mmap_addr == MAP_FAILED) {
1272                 VHOST_LOG_CONFIG(ERR, "(%s) mmap failed (%s).\n", dev->ifname, strerror(errno));
1273                 return -1;
1274         }
1275
1276         region->mmap_addr = mmap_addr;
1277         region->mmap_size = mmap_size;
1278         region->host_user_addr = (uint64_t)(uintptr_t)mmap_addr + mmap_offset;
1279
1280         if (dev->async_copy) {
1281                 if (add_guest_pages(dev, region, alignment) < 0) {
1282                         VHOST_LOG_CONFIG(ERR, "(%s) adding guest pages to region failed.\n",
1283                                         dev->ifname);
1284                         return -1;
1285                 }
1286         }
1287
1288         VHOST_LOG_CONFIG(INFO, "(%s) guest memory region size: 0x%" PRIx64 "\n",
1289                         dev->ifname, region->size);
1290         VHOST_LOG_CONFIG(INFO, "(%s)\t guest physical addr: 0x%" PRIx64 "\n",
1291                         dev->ifname, region->guest_phys_addr);
1292         VHOST_LOG_CONFIG(INFO, "(%s)\t guest virtual  addr: 0x%" PRIx64 "\n",
1293                         dev->ifname, region->guest_user_addr);
1294         VHOST_LOG_CONFIG(INFO, "(%s)\t host  virtual  addr: 0x%" PRIx64 "\n",
1295                         dev->ifname, region->host_user_addr);
1296         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap addr : 0x%" PRIx64 "\n",
1297                         dev->ifname, (uint64_t)(uintptr_t)mmap_addr);
1298         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap size : 0x%" PRIx64 "\n",
1299                         dev->ifname, mmap_size);
1300         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap align: 0x%" PRIx64 "\n",
1301                         dev->ifname, alignment);
1302         VHOST_LOG_CONFIG(INFO, "(%s)\t mmap off  : 0x%" PRIx64 "\n",
1303                         dev->ifname, mmap_offset);
1304
1305         return 0;
1306 }
1307
1308 static int
1309 vhost_user_set_mem_table(struct virtio_net **pdev,
1310                         struct vhu_msg_context *ctx,
1311                         int main_fd)
1312 {
1313         struct virtio_net *dev = *pdev;
1314         struct VhostUserMemory *memory = &ctx->msg.payload.memory;
1315         struct rte_vhost_mem_region *reg;
1316         int numa_node = SOCKET_ID_ANY;
1317         uint64_t mmap_offset;
1318         uint32_t i;
1319         bool async_notify = false;
1320
1321         if (validate_msg_fds(dev, ctx, memory->nregions) != 0)
1322                 return RTE_VHOST_MSG_RESULT_ERR;
1323
1324         if (memory->nregions > VHOST_MEMORY_MAX_NREGIONS) {
1325                 VHOST_LOG_CONFIG(ERR, "(%s) too many memory regions (%u)\n",
1326                                 dev->ifname, memory->nregions);
1327                 goto close_msg_fds;
1328         }
1329
1330         if (dev->mem && !vhost_memory_changed(memory, dev->mem)) {
1331                 VHOST_LOG_CONFIG(INFO, "(%s) memory regions not changed\n", dev->ifname);
1332
1333                 close_msg_fds(ctx);
1334
1335                 return RTE_VHOST_MSG_RESULT_OK;
1336         }
1337
1338         if (dev->mem) {
1339                 if (dev->flags & VIRTIO_DEV_VDPA_CONFIGURED) {
1340                         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
1341
1342                         if (vdpa_dev && vdpa_dev->ops->dev_close)
1343                                 vdpa_dev->ops->dev_close(dev->vid);
1344                         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
1345                 }
1346
1347                 /* notify the vhost application to stop DMA transfers */
1348                 if (dev->async_copy && dev->notify_ops->vring_state_changed) {
1349                         for (i = 0; i < dev->nr_vring; i++) {
1350                                 dev->notify_ops->vring_state_changed(dev->vid,
1351                                                 i, 0);
1352                         }
1353                         async_notify = true;
1354                 }
1355
1356                 free_mem_region(dev);
1357                 rte_free(dev->mem);
1358                 dev->mem = NULL;
1359         }
1360
1361         /* Flush IOTLB cache as previous HVAs are now invalid */
1362         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1363                 for (i = 0; i < dev->nr_vring; i++)
1364                         vhost_user_iotlb_flush_all(dev->virtqueue[i]);
1365
1366         /*
1367          * If VQ 0 has already been allocated, try to allocate on the same
1368          * NUMA node. It can be reallocated later in numa_realloc().
1369          */
1370         if (dev->nr_vring > 0)
1371                 numa_node = dev->virtqueue[0]->numa_node;
1372
1373         dev->nr_guest_pages = 0;
1374         if (dev->guest_pages == NULL) {
1375                 dev->max_guest_pages = 8;
1376                 dev->guest_pages = rte_zmalloc_socket(NULL,
1377                                         dev->max_guest_pages *
1378                                         sizeof(struct guest_page),
1379                                         RTE_CACHE_LINE_SIZE,
1380                                         numa_node);
1381                 if (dev->guest_pages == NULL) {
1382                         VHOST_LOG_CONFIG(ERR,
1383                                 "(%s) failed to allocate memory for dev->guest_pages\n",
1384                                 dev->ifname);
1385                         goto close_msg_fds;
1386                 }
1387         }
1388
1389         dev->mem = rte_zmalloc_socket("vhost-mem-table", sizeof(struct rte_vhost_memory) +
1390                 sizeof(struct rte_vhost_mem_region) * memory->nregions, 0, numa_node);
1391         if (dev->mem == NULL) {
1392                 VHOST_LOG_CONFIG(ERR,
1393                         "(%s) failed to allocate memory for dev->mem\n",
1394                         dev->ifname);
1395                 goto free_guest_pages;
1396         }
1397
1398         for (i = 0; i < memory->nregions; i++) {
1399                 reg = &dev->mem->regions[i];
1400
1401                 reg->guest_phys_addr = memory->regions[i].guest_phys_addr;
1402                 reg->guest_user_addr = memory->regions[i].userspace_addr;
1403                 reg->size            = memory->regions[i].memory_size;
1404                 reg->fd              = ctx->fds[i];
1405
1406                 /*
1407                  * Assign invalid file descriptor value to avoid double
1408                  * closing on error path.
1409                  */
1410                 ctx->fds[i] = -1;
1411
1412                 mmap_offset = memory->regions[i].mmap_offset;
1413
1414                 if (vhost_user_mmap_region(dev, reg, mmap_offset) < 0) {
1415                         VHOST_LOG_CONFIG(ERR, "(%s) failed to mmap region %u\n", dev->ifname, i);
1416                         goto free_mem_table;
1417                 }
1418
1419                 dev->mem->nregions++;
1420         }
1421
1422         if (dev->async_copy && rte_vfio_is_enabled("vfio"))
1423                 async_dma_map(dev, true);
1424
1425         if (vhost_user_postcopy_register(dev, main_fd, ctx) < 0)
1426                 goto free_mem_table;
1427
1428         for (i = 0; i < dev->nr_vring; i++) {
1429                 struct vhost_virtqueue *vq = dev->virtqueue[i];
1430
1431                 if (!vq)
1432                         continue;
1433
1434                 if (vq->desc || vq->avail || vq->used) {
1435                         /*
1436                          * If the memory table got updated, the ring addresses
1437                          * need to be translated again as virtual addresses have
1438                          * changed.
1439                          */
1440                         vring_invalidate(dev, vq);
1441
1442                         dev = translate_ring_addresses(dev, i);
1443                         if (!dev) {
1444                                 dev = *pdev;
1445                                 goto free_mem_table;
1446                         }
1447
1448                         *pdev = dev;
1449                 }
1450         }
1451
1452         dump_guest_pages(dev);
1453
1454         if (async_notify) {
1455                 for (i = 0; i < dev->nr_vring; i++)
1456                         dev->notify_ops->vring_state_changed(dev->vid, i, 1);
1457         }
1458
1459         return RTE_VHOST_MSG_RESULT_OK;
1460
1461 free_mem_table:
1462         free_mem_region(dev);
1463         rte_free(dev->mem);
1464         dev->mem = NULL;
1465
1466 free_guest_pages:
1467         rte_free(dev->guest_pages);
1468         dev->guest_pages = NULL;
1469 close_msg_fds:
1470         close_msg_fds(ctx);
1471         return RTE_VHOST_MSG_RESULT_ERR;
1472 }
1473
1474 static bool
1475 vq_is_ready(struct virtio_net *dev, struct vhost_virtqueue *vq)
1476 {
1477         bool rings_ok;
1478
1479         if (!vq)
1480                 return false;
1481
1482         if (vq_is_packed(dev))
1483                 rings_ok = vq->desc_packed && vq->driver_event &&
1484                         vq->device_event;
1485         else
1486                 rings_ok = vq->desc && vq->avail && vq->used;
1487
1488         return rings_ok &&
1489                vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1490                vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1491                vq->enabled;
1492 }
1493
1494 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
1495
1496 static int
1497 virtio_is_ready(struct virtio_net *dev)
1498 {
1499         struct vhost_virtqueue *vq;
1500         uint32_t i, nr_vring = dev->nr_vring;
1501
1502         if (dev->flags & VIRTIO_DEV_READY)
1503                 return 1;
1504
1505         if (!dev->nr_vring)
1506                 return 0;
1507
1508         if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
1509                 nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
1510
1511                 if (dev->nr_vring < nr_vring)
1512                         return 0;
1513         }
1514
1515         for (i = 0; i < nr_vring; i++) {
1516                 vq = dev->virtqueue[i];
1517
1518                 if (!vq_is_ready(dev, vq))
1519                         return 0;
1520         }
1521
1522         /* If supported, ensure the frontend is really done with config */
1523         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
1524                 if (!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK))
1525                         return 0;
1526
1527         dev->flags |= VIRTIO_DEV_READY;
1528
1529         if (!(dev->flags & VIRTIO_DEV_RUNNING))
1530                 VHOST_LOG_CONFIG(INFO, "(%s) virtio is now ready for processing.\n", dev->ifname);
1531         return 1;
1532 }
1533
1534 static void *
1535 inflight_mem_alloc(struct virtio_net *dev, const char *name, size_t size, int *fd)
1536 {
1537         void *ptr;
1538         int mfd = -1;
1539         char fname[20] = "/tmp/memfd-XXXXXX";
1540
1541         *fd = -1;
1542 #ifdef MEMFD_SUPPORTED
1543         mfd = memfd_create(name, MFD_CLOEXEC);
1544 #else
1545         RTE_SET_USED(name);
1546 #endif
1547         if (mfd == -1) {
1548                 mfd = mkstemp(fname);
1549                 if (mfd == -1) {
1550                         VHOST_LOG_CONFIG(ERR, "(%s) failed to get inflight buffer fd\n",
1551                                         dev->ifname);
1552                         return NULL;
1553                 }
1554
1555                 unlink(fname);
1556         }
1557
1558         if (ftruncate(mfd, size) == -1) {
1559                 VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc inflight buffer\n", dev->ifname);
1560                 close(mfd);
1561                 return NULL;
1562         }
1563
1564         ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
1565         if (ptr == MAP_FAILED) {
1566                 VHOST_LOG_CONFIG(ERR, "(%s) failed to mmap inflight buffer\n", dev->ifname);
1567                 close(mfd);
1568                 return NULL;
1569         }
1570
1571         *fd = mfd;
1572         return ptr;
1573 }
1574
1575 static uint32_t
1576 get_pervq_shm_size_split(uint16_t queue_size)
1577 {
1578         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_split) *
1579                                   queue_size + sizeof(uint64_t) +
1580                                   sizeof(uint16_t) * 4, INFLIGHT_ALIGNMENT);
1581 }
1582
1583 static uint32_t
1584 get_pervq_shm_size_packed(uint16_t queue_size)
1585 {
1586         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_packed)
1587                                   * queue_size + sizeof(uint64_t) +
1588                                   sizeof(uint16_t) * 6 + sizeof(uint8_t) * 9,
1589                                   INFLIGHT_ALIGNMENT);
1590 }
1591
1592 static int
1593 vhost_user_get_inflight_fd(struct virtio_net **pdev,
1594                            struct vhu_msg_context *ctx,
1595                            int main_fd __rte_unused)
1596 {
1597         struct rte_vhost_inflight_info_packed *inflight_packed;
1598         uint64_t pervq_inflight_size, mmap_size;
1599         uint16_t num_queues, queue_size;
1600         struct virtio_net *dev = *pdev;
1601         int fd, i, j;
1602         int numa_node = SOCKET_ID_ANY;
1603         void *addr;
1604
1605         if (validate_msg_fds(dev, ctx, 0) != 0)
1606                 return RTE_VHOST_MSG_RESULT_ERR;
1607
1608         if (ctx->msg.size != sizeof(ctx->msg.payload.inflight)) {
1609                 VHOST_LOG_CONFIG(ERR, "(%s) invalid get_inflight_fd message size is %d\n",
1610                         dev->ifname, ctx->msg.size);
1611                 return RTE_VHOST_MSG_RESULT_ERR;
1612         }
1613
1614         /*
1615          * If VQ 0 has already been allocated, try to allocate on the same
1616          * NUMA node. It can be reallocated later in numa_realloc().
1617          */
1618         if (dev->nr_vring > 0)
1619                 numa_node = dev->virtqueue[0]->numa_node;
1620
1621         if (dev->inflight_info == NULL) {
1622                 dev->inflight_info = rte_zmalloc_socket("inflight_info",
1623                                 sizeof(struct inflight_mem_info), 0, numa_node);
1624                 if (!dev->inflight_info) {
1625                         VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc dev inflight area\n",
1626                                         dev->ifname);
1627                         return RTE_VHOST_MSG_RESULT_ERR;
1628                 }
1629                 dev->inflight_info->fd = -1;
1630         }
1631
1632         num_queues = ctx->msg.payload.inflight.num_queues;
1633         queue_size = ctx->msg.payload.inflight.queue_size;
1634
1635         VHOST_LOG_CONFIG(INFO, "(%s) get_inflight_fd num_queues: %u\n",
1636                 dev->ifname, ctx->msg.payload.inflight.num_queues);
1637         VHOST_LOG_CONFIG(INFO, "(%s) get_inflight_fd queue_size: %u\n",
1638                 dev->ifname, ctx->msg.payload.inflight.queue_size);
1639
1640         if (vq_is_packed(dev))
1641                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1642         else
1643                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1644
1645         mmap_size = num_queues * pervq_inflight_size;
1646         addr = inflight_mem_alloc(dev, "vhost-inflight", mmap_size, &fd);
1647         if (!addr) {
1648                 VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc vhost inflight area\n", dev->ifname);
1649                         ctx->msg.payload.inflight.mmap_size = 0;
1650                 return RTE_VHOST_MSG_RESULT_ERR;
1651         }
1652         memset(addr, 0, mmap_size);
1653
1654         if (dev->inflight_info->addr) {
1655                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1656                 dev->inflight_info->addr = NULL;
1657         }
1658
1659         if (dev->inflight_info->fd >= 0) {
1660                 close(dev->inflight_info->fd);
1661                 dev->inflight_info->fd = -1;
1662         }
1663
1664         dev->inflight_info->addr = addr;
1665         dev->inflight_info->size = ctx->msg.payload.inflight.mmap_size = mmap_size;
1666         dev->inflight_info->fd = ctx->fds[0] = fd;
1667         ctx->msg.payload.inflight.mmap_offset = 0;
1668         ctx->fd_num = 1;
1669
1670         if (vq_is_packed(dev)) {
1671                 for (i = 0; i < num_queues; i++) {
1672                         inflight_packed =
1673                                 (struct rte_vhost_inflight_info_packed *)addr;
1674                         inflight_packed->used_wrap_counter = 1;
1675                         inflight_packed->old_used_wrap_counter = 1;
1676                         for (j = 0; j < queue_size; j++)
1677                                 inflight_packed->desc[j].next = j + 1;
1678                         addr = (void *)((char *)addr + pervq_inflight_size);
1679                 }
1680         }
1681
1682         VHOST_LOG_CONFIG(INFO, "(%s) send inflight mmap_size: %"PRIu64"\n",
1683                         dev->ifname, ctx->msg.payload.inflight.mmap_size);
1684         VHOST_LOG_CONFIG(INFO, "(%s) send inflight mmap_offset: %"PRIu64"\n",
1685                         dev->ifname, ctx->msg.payload.inflight.mmap_offset);
1686         VHOST_LOG_CONFIG(INFO, "(%s) send inflight fd: %d\n", dev->ifname, ctx->fds[0]);
1687
1688         return RTE_VHOST_MSG_RESULT_REPLY;
1689 }
1690
1691 static int
1692 vhost_user_set_inflight_fd(struct virtio_net **pdev,
1693                            struct vhu_msg_context *ctx,
1694                            int main_fd __rte_unused)
1695 {
1696         uint64_t mmap_size, mmap_offset;
1697         uint16_t num_queues, queue_size;
1698         struct virtio_net *dev = *pdev;
1699         uint32_t pervq_inflight_size;
1700         struct vhost_virtqueue *vq;
1701         void *addr;
1702         int fd, i;
1703         int numa_node = SOCKET_ID_ANY;
1704
1705         if (validate_msg_fds(dev, ctx, 1) != 0)
1706                 return RTE_VHOST_MSG_RESULT_ERR;
1707
1708         fd = ctx->fds[0];
1709         if (ctx->msg.size != sizeof(ctx->msg.payload.inflight) || fd < 0) {
1710                 VHOST_LOG_CONFIG(ERR, "(%s) invalid set_inflight_fd message size is %d,fd is %d\n",
1711                         dev->ifname, ctx->msg.size, fd);
1712                 return RTE_VHOST_MSG_RESULT_ERR;
1713         }
1714
1715         mmap_size = ctx->msg.payload.inflight.mmap_size;
1716         mmap_offset = ctx->msg.payload.inflight.mmap_offset;
1717         num_queues = ctx->msg.payload.inflight.num_queues;
1718         queue_size = ctx->msg.payload.inflight.queue_size;
1719
1720         if (vq_is_packed(dev))
1721                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1722         else
1723                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1724
1725         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd mmap_size: %"PRIu64"\n",
1726                         dev->ifname, mmap_size);
1727         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd mmap_offset: %"PRIu64"\n",
1728                         dev->ifname, mmap_offset);
1729         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd num_queues: %u\n", dev->ifname, num_queues);
1730         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd queue_size: %u\n", dev->ifname, queue_size);
1731         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd fd: %d\n", dev->ifname, fd);
1732         VHOST_LOG_CONFIG(INFO, "(%s) set_inflight_fd pervq_inflight_size: %d\n",
1733                         dev->ifname, pervq_inflight_size);
1734
1735         /*
1736          * If VQ 0 has already been allocated, try to allocate on the same
1737          * NUMA node. It can be reallocated later in numa_realloc().
1738          */
1739         if (dev->nr_vring > 0)
1740                 numa_node = dev->virtqueue[0]->numa_node;
1741
1742         if (!dev->inflight_info) {
1743                 dev->inflight_info = rte_zmalloc_socket("inflight_info",
1744                                 sizeof(struct inflight_mem_info), 0, numa_node);
1745                 if (dev->inflight_info == NULL) {
1746                         VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc dev inflight area\n",
1747                                         dev->ifname);
1748                         return RTE_VHOST_MSG_RESULT_ERR;
1749                 }
1750                 dev->inflight_info->fd = -1;
1751         }
1752
1753         if (dev->inflight_info->addr) {
1754                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1755                 dev->inflight_info->addr = NULL;
1756         }
1757
1758         addr = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1759                     fd, mmap_offset);
1760         if (addr == MAP_FAILED) {
1761                 VHOST_LOG_CONFIG(ERR, "(%s) failed to mmap share memory.\n", dev->ifname);
1762                 return RTE_VHOST_MSG_RESULT_ERR;
1763         }
1764
1765         if (dev->inflight_info->fd >= 0) {
1766                 close(dev->inflight_info->fd);
1767                 dev->inflight_info->fd = -1;
1768         }
1769
1770         dev->inflight_info->fd = fd;
1771         dev->inflight_info->addr = addr;
1772         dev->inflight_info->size = mmap_size;
1773
1774         for (i = 0; i < num_queues; i++) {
1775                 vq = dev->virtqueue[i];
1776                 if (!vq)
1777                         continue;
1778
1779                 if (vq_is_packed(dev)) {
1780                         vq->inflight_packed = addr;
1781                         vq->inflight_packed->desc_num = queue_size;
1782                 } else {
1783                         vq->inflight_split = addr;
1784                         vq->inflight_split->desc_num = queue_size;
1785                 }
1786                 addr = (void *)((char *)addr + pervq_inflight_size);
1787         }
1788
1789         return RTE_VHOST_MSG_RESULT_OK;
1790 }
1791
1792 static int
1793 vhost_user_set_vring_call(struct virtio_net **pdev,
1794                         struct vhu_msg_context *ctx,
1795                         int main_fd __rte_unused)
1796 {
1797         struct virtio_net *dev = *pdev;
1798         struct vhost_vring_file file;
1799         struct vhost_virtqueue *vq;
1800         int expected_fds;
1801
1802         expected_fds = (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1803         if (validate_msg_fds(dev, ctx, expected_fds) != 0)
1804                 return RTE_VHOST_MSG_RESULT_ERR;
1805
1806         file.index = ctx->msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
1807         if (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1808                 file.fd = VIRTIO_INVALID_EVENTFD;
1809         else
1810                 file.fd = ctx->fds[0];
1811         VHOST_LOG_CONFIG(INFO, "(%s) vring call idx:%d file:%d\n",
1812                         dev->ifname, file.index, file.fd);
1813
1814         vq = dev->virtqueue[file.index];
1815
1816         if (vq->ready) {
1817                 vq->ready = false;
1818                 vhost_user_notify_queue_state(dev, file.index, 0);
1819         }
1820
1821         if (vq->callfd >= 0)
1822                 close(vq->callfd);
1823
1824         vq->callfd = file.fd;
1825
1826         return RTE_VHOST_MSG_RESULT_OK;
1827 }
1828
1829 static int vhost_user_set_vring_err(struct virtio_net **pdev,
1830                         struct vhu_msg_context *ctx,
1831                         int main_fd __rte_unused)
1832 {
1833         struct virtio_net *dev = *pdev;
1834         int expected_fds;
1835
1836         expected_fds = (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1837         if (validate_msg_fds(dev, ctx, expected_fds) != 0)
1838                 return RTE_VHOST_MSG_RESULT_ERR;
1839
1840         if (!(ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1841                 close(ctx->fds[0]);
1842         VHOST_LOG_CONFIG(INFO, "(%s) not implemented\n", dev->ifname);
1843
1844         return RTE_VHOST_MSG_RESULT_OK;
1845 }
1846
1847 static int
1848 resubmit_desc_compare(const void *a, const void *b)
1849 {
1850         const struct rte_vhost_resubmit_desc *desc0 = a;
1851         const struct rte_vhost_resubmit_desc *desc1 = b;
1852
1853         if (desc1->counter > desc0->counter)
1854                 return 1;
1855
1856         return -1;
1857 }
1858
1859 static int
1860 vhost_check_queue_inflights_split(struct virtio_net *dev,
1861                                   struct vhost_virtqueue *vq)
1862 {
1863         uint16_t i;
1864         uint16_t resubmit_num = 0, last_io, num;
1865         struct vring_used *used = vq->used;
1866         struct rte_vhost_resubmit_info *resubmit;
1867         struct rte_vhost_inflight_info_split *inflight_split;
1868
1869         if (!(dev->protocol_features &
1870             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1871                 return RTE_VHOST_MSG_RESULT_OK;
1872
1873         /* The frontend may still not support the inflight feature
1874          * although we negotiate the protocol feature.
1875          */
1876         if ((!vq->inflight_split))
1877                 return RTE_VHOST_MSG_RESULT_OK;
1878
1879         if (!vq->inflight_split->version) {
1880                 vq->inflight_split->version = INFLIGHT_VERSION;
1881                 return RTE_VHOST_MSG_RESULT_OK;
1882         }
1883
1884         if (vq->resubmit_inflight)
1885                 return RTE_VHOST_MSG_RESULT_OK;
1886
1887         inflight_split = vq->inflight_split;
1888         vq->global_counter = 0;
1889         last_io = inflight_split->last_inflight_io;
1890
1891         if (inflight_split->used_idx != used->idx) {
1892                 inflight_split->desc[last_io].inflight = 0;
1893                 rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
1894                 inflight_split->used_idx = used->idx;
1895         }
1896
1897         for (i = 0; i < inflight_split->desc_num; i++) {
1898                 if (inflight_split->desc[i].inflight == 1)
1899                         resubmit_num++;
1900         }
1901
1902         vq->last_avail_idx += resubmit_num;
1903
1904         if (resubmit_num) {
1905                 resubmit = rte_zmalloc_socket("resubmit", sizeof(struct rte_vhost_resubmit_info),
1906                                 0, vq->numa_node);
1907                 if (!resubmit) {
1908                         VHOST_LOG_CONFIG(ERR,
1909                                         "(%s) failed to allocate memory for resubmit info.\n",
1910                                         dev->ifname);
1911                         return RTE_VHOST_MSG_RESULT_ERR;
1912                 }
1913
1914                 resubmit->resubmit_list = rte_zmalloc_socket("resubmit_list",
1915                                 resubmit_num * sizeof(struct rte_vhost_resubmit_desc),
1916                                 0, vq->numa_node);
1917                 if (!resubmit->resubmit_list) {
1918                         VHOST_LOG_CONFIG(ERR,
1919                                         "(%s) failed to allocate memory for inflight desc.\n",
1920                                         dev->ifname);
1921                         rte_free(resubmit);
1922                         return RTE_VHOST_MSG_RESULT_ERR;
1923                 }
1924
1925                 num = 0;
1926                 for (i = 0; i < vq->inflight_split->desc_num; i++) {
1927                         if (vq->inflight_split->desc[i].inflight == 1) {
1928                                 resubmit->resubmit_list[num].index = i;
1929                                 resubmit->resubmit_list[num].counter =
1930                                         inflight_split->desc[i].counter;
1931                                 num++;
1932                         }
1933                 }
1934                 resubmit->resubmit_num = num;
1935
1936                 if (resubmit->resubmit_num > 1)
1937                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1938                               sizeof(struct rte_vhost_resubmit_desc),
1939                               resubmit_desc_compare);
1940
1941                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1942                 vq->resubmit_inflight = resubmit;
1943         }
1944
1945         return RTE_VHOST_MSG_RESULT_OK;
1946 }
1947
1948 static int
1949 vhost_check_queue_inflights_packed(struct virtio_net *dev,
1950                                    struct vhost_virtqueue *vq)
1951 {
1952         uint16_t i;
1953         uint16_t resubmit_num = 0, old_used_idx, num;
1954         struct rte_vhost_resubmit_info *resubmit;
1955         struct rte_vhost_inflight_info_packed *inflight_packed;
1956
1957         if (!(dev->protocol_features &
1958             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1959                 return RTE_VHOST_MSG_RESULT_OK;
1960
1961         /* The frontend may still not support the inflight feature
1962          * although we negotiate the protocol feature.
1963          */
1964         if ((!vq->inflight_packed))
1965                 return RTE_VHOST_MSG_RESULT_OK;
1966
1967         if (!vq->inflight_packed->version) {
1968                 vq->inflight_packed->version = INFLIGHT_VERSION;
1969                 return RTE_VHOST_MSG_RESULT_OK;
1970         }
1971
1972         if (vq->resubmit_inflight)
1973                 return RTE_VHOST_MSG_RESULT_OK;
1974
1975         inflight_packed = vq->inflight_packed;
1976         vq->global_counter = 0;
1977         old_used_idx = inflight_packed->old_used_idx;
1978
1979         if (inflight_packed->used_idx != old_used_idx) {
1980                 if (inflight_packed->desc[old_used_idx].inflight == 0) {
1981                         inflight_packed->old_used_idx =
1982                                 inflight_packed->used_idx;
1983                         inflight_packed->old_used_wrap_counter =
1984                                 inflight_packed->used_wrap_counter;
1985                         inflight_packed->old_free_head =
1986                                 inflight_packed->free_head;
1987                 } else {
1988                         inflight_packed->used_idx =
1989                                 inflight_packed->old_used_idx;
1990                         inflight_packed->used_wrap_counter =
1991                                 inflight_packed->old_used_wrap_counter;
1992                         inflight_packed->free_head =
1993                                 inflight_packed->old_free_head;
1994                 }
1995         }
1996
1997         for (i = 0; i < inflight_packed->desc_num; i++) {
1998                 if (inflight_packed->desc[i].inflight == 1)
1999                         resubmit_num++;
2000         }
2001
2002         if (resubmit_num) {
2003                 resubmit = rte_zmalloc_socket("resubmit", sizeof(struct rte_vhost_resubmit_info),
2004                                 0, vq->numa_node);
2005                 if (resubmit == NULL) {
2006                         VHOST_LOG_CONFIG(ERR,
2007                                         "(%s) failed to allocate memory for resubmit info.\n",
2008                                         dev->ifname);
2009                         return RTE_VHOST_MSG_RESULT_ERR;
2010                 }
2011
2012                 resubmit->resubmit_list = rte_zmalloc_socket("resubmit_list",
2013                                 resubmit_num * sizeof(struct rte_vhost_resubmit_desc),
2014                                 0, vq->numa_node);
2015                 if (resubmit->resubmit_list == NULL) {
2016                         VHOST_LOG_CONFIG(ERR,
2017                                         "(%s) failed to allocate memory for resubmit desc.\n",
2018                                         dev->ifname);
2019                         rte_free(resubmit);
2020                         return RTE_VHOST_MSG_RESULT_ERR;
2021                 }
2022
2023                 num = 0;
2024                 for (i = 0; i < inflight_packed->desc_num; i++) {
2025                         if (vq->inflight_packed->desc[i].inflight == 1) {
2026                                 resubmit->resubmit_list[num].index = i;
2027                                 resubmit->resubmit_list[num].counter =
2028                                         inflight_packed->desc[i].counter;
2029                                 num++;
2030                         }
2031                 }
2032                 resubmit->resubmit_num = num;
2033
2034                 if (resubmit->resubmit_num > 1)
2035                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
2036                               sizeof(struct rte_vhost_resubmit_desc),
2037                               resubmit_desc_compare);
2038
2039                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
2040                 vq->resubmit_inflight = resubmit;
2041         }
2042
2043         return RTE_VHOST_MSG_RESULT_OK;
2044 }
2045
2046 static int
2047 vhost_user_set_vring_kick(struct virtio_net **pdev,
2048                         struct vhu_msg_context *ctx,
2049                         int main_fd __rte_unused)
2050 {
2051         struct virtio_net *dev = *pdev;
2052         struct vhost_vring_file file;
2053         struct vhost_virtqueue *vq;
2054         int expected_fds;
2055
2056         expected_fds = (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
2057         if (validate_msg_fds(dev, ctx, expected_fds) != 0)
2058                 return RTE_VHOST_MSG_RESULT_ERR;
2059
2060         file.index = ctx->msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
2061         if (ctx->msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)
2062                 file.fd = VIRTIO_INVALID_EVENTFD;
2063         else
2064                 file.fd = ctx->fds[0];
2065         VHOST_LOG_CONFIG(INFO, "(%s) vring kick idx:%d file:%d\n",
2066                         dev->ifname, file.index, file.fd);
2067
2068         /* Interpret ring addresses only when ring is started. */
2069         dev = translate_ring_addresses(dev, file.index);
2070         if (!dev) {
2071                 if (file.fd != VIRTIO_INVALID_EVENTFD)
2072                         close(file.fd);
2073
2074                 return RTE_VHOST_MSG_RESULT_ERR;
2075         }
2076
2077         *pdev = dev;
2078
2079         vq = dev->virtqueue[file.index];
2080
2081         /*
2082          * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated,
2083          * the ring starts already enabled. Otherwise, it is enabled via
2084          * the SET_VRING_ENABLE message.
2085          */
2086         if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) {
2087                 vq->enabled = true;
2088         }
2089
2090         if (vq->ready) {
2091                 vq->ready = false;
2092                 vhost_user_notify_queue_state(dev, file.index, 0);
2093         }
2094
2095         if (vq->kickfd >= 0)
2096                 close(vq->kickfd);
2097         vq->kickfd = file.fd;
2098
2099         if (vq_is_packed(dev)) {
2100                 if (vhost_check_queue_inflights_packed(dev, vq)) {
2101                         VHOST_LOG_CONFIG(ERR, "(%s) failed to inflights for vq: %d\n",
2102                                         dev->ifname, file.index);
2103                         return RTE_VHOST_MSG_RESULT_ERR;
2104                 }
2105         } else {
2106                 if (vhost_check_queue_inflights_split(dev, vq)) {
2107                         VHOST_LOG_CONFIG(ERR, "(%s) failed to inflights for vq: %d\n",
2108                                         dev->ifname, file.index);
2109                         return RTE_VHOST_MSG_RESULT_ERR;
2110                 }
2111         }
2112
2113         return RTE_VHOST_MSG_RESULT_OK;
2114 }
2115
2116 /*
2117  * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
2118  */
2119 static int
2120 vhost_user_get_vring_base(struct virtio_net **pdev,
2121                         struct vhu_msg_context *ctx,
2122                         int main_fd __rte_unused)
2123 {
2124         struct virtio_net *dev = *pdev;
2125         struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index];
2126         uint64_t val;
2127
2128         if (validate_msg_fds(dev, ctx, 0) != 0)
2129                 return RTE_VHOST_MSG_RESULT_ERR;
2130
2131         /* We have to stop the queue (virtio) if it is running. */
2132         vhost_destroy_device_notify(dev);
2133
2134         dev->flags &= ~VIRTIO_DEV_READY;
2135         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
2136
2137         /* Here we are safe to get the indexes */
2138         if (vq_is_packed(dev)) {
2139                 /*
2140                  * Bit[0:14]: avail index
2141                  * Bit[15]: avail wrap counter
2142                  */
2143                 val = vq->last_avail_idx & 0x7fff;
2144                 val |= vq->avail_wrap_counter << 15;
2145                 ctx->msg.payload.state.num = val;
2146         } else {
2147                 ctx->msg.payload.state.num = vq->last_avail_idx;
2148         }
2149
2150         VHOST_LOG_CONFIG(INFO, "(%s) vring base idx:%d file:%d\n",
2151                         dev->ifname, ctx->msg.payload.state.index,
2152                         ctx->msg.payload.state.num);
2153         /*
2154          * Based on current qemu vhost-user implementation, this message is
2155          * sent and only sent in vhost_vring_stop.
2156          * TODO: cleanup the vring, it isn't usable since here.
2157          */
2158         if (vq->kickfd >= 0)
2159                 close(vq->kickfd);
2160
2161         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
2162
2163         if (vq->callfd >= 0)
2164                 close(vq->callfd);
2165
2166         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
2167
2168         vq->signalled_used_valid = false;
2169
2170         if (vq_is_packed(dev)) {
2171                 rte_free(vq->shadow_used_packed);
2172                 vq->shadow_used_packed = NULL;
2173         } else {
2174                 rte_free(vq->shadow_used_split);
2175                 vq->shadow_used_split = NULL;
2176         }
2177
2178         rte_free(vq->batch_copy_elems);
2179         vq->batch_copy_elems = NULL;
2180
2181         rte_free(vq->log_cache);
2182         vq->log_cache = NULL;
2183
2184         ctx->msg.size = sizeof(ctx->msg.payload.state);
2185         ctx->fd_num = 0;
2186
2187         vhost_user_iotlb_flush_all(vq);
2188
2189         vring_invalidate(dev, vq);
2190
2191         return RTE_VHOST_MSG_RESULT_REPLY;
2192 }
2193
2194 /*
2195  * when virtio queues are ready to work, qemu will send us to
2196  * enable the virtio queue pair.
2197  */
2198 static int
2199 vhost_user_set_vring_enable(struct virtio_net **pdev,
2200                         struct vhu_msg_context *ctx,
2201                         int main_fd __rte_unused)
2202 {
2203         struct virtio_net *dev = *pdev;
2204         bool enable = !!ctx->msg.payload.state.num;
2205         int index = (int)ctx->msg.payload.state.index;
2206
2207         if (validate_msg_fds(dev, ctx, 0) != 0)
2208                 return RTE_VHOST_MSG_RESULT_ERR;
2209
2210         VHOST_LOG_CONFIG(INFO, "(%s) set queue enable: %d to qp idx: %d\n",
2211                         dev->ifname, enable, index);
2212
2213         if (enable && dev->virtqueue[index]->async) {
2214                 if (dev->virtqueue[index]->async->pkts_inflight_n) {
2215                         VHOST_LOG_CONFIG(ERR,
2216                                 "(%s) failed to enable vring. Inflight packets must be completed first\n",
2217                                 dev->ifname);
2218                         return RTE_VHOST_MSG_RESULT_ERR;
2219                 }
2220         }
2221
2222         dev->virtqueue[index]->enabled = enable;
2223
2224         return RTE_VHOST_MSG_RESULT_OK;
2225 }
2226
2227 static int
2228 vhost_user_get_protocol_features(struct virtio_net **pdev,
2229                         struct vhu_msg_context *ctx,
2230                         int main_fd __rte_unused)
2231 {
2232         struct virtio_net *dev = *pdev;
2233         uint64_t features, protocol_features;
2234
2235         if (validate_msg_fds(dev, ctx, 0) != 0)
2236                 return RTE_VHOST_MSG_RESULT_ERR;
2237
2238         rte_vhost_driver_get_features(dev->ifname, &features);
2239         rte_vhost_driver_get_protocol_features(dev->ifname, &protocol_features);
2240
2241         ctx->msg.payload.u64 = protocol_features;
2242         ctx->msg.size = sizeof(ctx->msg.payload.u64);
2243         ctx->fd_num = 0;
2244
2245         return RTE_VHOST_MSG_RESULT_REPLY;
2246 }
2247
2248 static int
2249 vhost_user_set_protocol_features(struct virtio_net **pdev,
2250                         struct vhu_msg_context *ctx,
2251                         int main_fd __rte_unused)
2252 {
2253         struct virtio_net *dev = *pdev;
2254         uint64_t protocol_features = ctx->msg.payload.u64;
2255         uint64_t slave_protocol_features = 0;
2256
2257         if (validate_msg_fds(dev, ctx, 0) != 0)
2258                 return RTE_VHOST_MSG_RESULT_ERR;
2259
2260         rte_vhost_driver_get_protocol_features(dev->ifname,
2261                         &slave_protocol_features);
2262         if (protocol_features & ~slave_protocol_features) {
2263                 VHOST_LOG_CONFIG(ERR, "(%s) received invalid protocol features.\n", dev->ifname);
2264                 return RTE_VHOST_MSG_RESULT_ERR;
2265         }
2266
2267         dev->protocol_features = protocol_features;
2268         VHOST_LOG_CONFIG(INFO, "(%s) negotiated Vhost-user protocol features: 0x%" PRIx64 "\n",
2269                 dev->ifname, dev->protocol_features);
2270
2271         return RTE_VHOST_MSG_RESULT_OK;
2272 }
2273
2274 static int
2275 vhost_user_set_log_base(struct virtio_net **pdev,
2276                         struct vhu_msg_context *ctx,
2277                         int main_fd __rte_unused)
2278 {
2279         struct virtio_net *dev = *pdev;
2280         int fd = ctx->fds[0];
2281         uint64_t size, off;
2282         void *addr;
2283         uint32_t i;
2284
2285         if (validate_msg_fds(dev, ctx, 1) != 0)
2286                 return RTE_VHOST_MSG_RESULT_ERR;
2287
2288         if (fd < 0) {
2289                 VHOST_LOG_CONFIG(ERR, "(%s) invalid log fd: %d\n", dev->ifname, fd);
2290                 return RTE_VHOST_MSG_RESULT_ERR;
2291         }
2292
2293         if (ctx->msg.size != sizeof(VhostUserLog)) {
2294                 VHOST_LOG_CONFIG(ERR, "(%s) invalid log base msg size: %"PRId32" != %d\n",
2295                         dev->ifname, ctx->msg.size, (int)sizeof(VhostUserLog));
2296                 goto close_msg_fds;
2297         }
2298
2299         size = ctx->msg.payload.log.mmap_size;
2300         off  = ctx->msg.payload.log.mmap_offset;
2301
2302         /* Check for mmap size and offset overflow. */
2303         if (off >= -size) {
2304                 VHOST_LOG_CONFIG(ERR,
2305                                 "(%s) log offset %#"PRIx64" and log size %#"PRIx64" overflow\n",
2306                                 dev->ifname, off, size);
2307                 goto close_msg_fds;
2308         }
2309
2310         VHOST_LOG_CONFIG(INFO, "(%s) log mmap size: %"PRId64", offset: %"PRId64"\n",
2311                         dev->ifname, size, off);
2312
2313         /*
2314          * mmap from 0 to workaround a hugepage mmap bug: mmap will
2315          * fail when offset is not page size aligned.
2316          */
2317         addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
2318         close(fd);
2319         if (addr == MAP_FAILED) {
2320                 VHOST_LOG_CONFIG(ERR, "(%s) mmap log base failed!\n", dev->ifname);
2321                 return RTE_VHOST_MSG_RESULT_ERR;
2322         }
2323
2324         /*
2325          * Free previously mapped log memory on occasionally
2326          * multiple VHOST_USER_SET_LOG_BASE.
2327          */
2328         if (dev->log_addr) {
2329                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
2330         }
2331         dev->log_addr = (uint64_t)(uintptr_t)addr;
2332         dev->log_base = dev->log_addr + off;
2333         dev->log_size = size;
2334
2335         for (i = 0; i < dev->nr_vring; i++) {
2336                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2337
2338                 rte_free(vq->log_cache);
2339                 vq->log_cache = NULL;
2340                 vq->log_cache_nb_elem = 0;
2341                 vq->log_cache = rte_malloc_socket("vq log cache",
2342                                 sizeof(struct log_cache_entry) * VHOST_LOG_CACHE_NR,
2343                                 0, vq->numa_node);
2344                 /*
2345                  * If log cache alloc fail, don't fail migration, but no
2346                  * caching will be done, which will impact performance
2347                  */
2348                 if (!vq->log_cache)
2349                         VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate VQ logging cache\n",
2350                                         dev->ifname);
2351         }
2352
2353         /*
2354          * The spec is not clear about it (yet), but QEMU doesn't expect
2355          * any payload in the reply.
2356          */
2357         ctx->msg.size = 0;
2358         ctx->fd_num = 0;
2359
2360         return RTE_VHOST_MSG_RESULT_REPLY;
2361
2362 close_msg_fds:
2363         close_msg_fds(ctx);
2364         return RTE_VHOST_MSG_RESULT_ERR;
2365 }
2366
2367 static int vhost_user_set_log_fd(struct virtio_net **pdev,
2368                         struct vhu_msg_context *ctx,
2369                         int main_fd __rte_unused)
2370 {
2371         struct virtio_net *dev = *pdev;
2372
2373         if (validate_msg_fds(dev, ctx, 1) != 0)
2374                 return RTE_VHOST_MSG_RESULT_ERR;
2375
2376         close(ctx->fds[0]);
2377         VHOST_LOG_CONFIG(INFO, "(%s) not implemented.\n", dev->ifname);
2378
2379         return RTE_VHOST_MSG_RESULT_OK;
2380 }
2381
2382 /*
2383  * An rarp packet is constructed and broadcasted to notify switches about
2384  * the new location of the migrated VM, so that packets from outside will
2385  * not be lost after migration.
2386  *
2387  * However, we don't actually "send" a rarp packet here, instead, we set
2388  * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
2389  */
2390 static int
2391 vhost_user_send_rarp(struct virtio_net **pdev,
2392                         struct vhu_msg_context *ctx,
2393                         int main_fd __rte_unused)
2394 {
2395         struct virtio_net *dev = *pdev;
2396         uint8_t *mac = (uint8_t *)&ctx->msg.payload.u64;
2397         struct rte_vdpa_device *vdpa_dev;
2398
2399         if (validate_msg_fds(dev, ctx, 0) != 0)
2400                 return RTE_VHOST_MSG_RESULT_ERR;
2401
2402         VHOST_LOG_CONFIG(DEBUG, "(%s) MAC: " RTE_ETHER_ADDR_PRT_FMT "\n",
2403                 dev->ifname, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
2404         memcpy(dev->mac.addr_bytes, mac, 6);
2405
2406         /*
2407          * Set the flag to inject a RARP broadcast packet at
2408          * rte_vhost_dequeue_burst().
2409          *
2410          * __ATOMIC_RELEASE ordering is for making sure the mac is
2411          * copied before the flag is set.
2412          */
2413         __atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
2414         vdpa_dev = dev->vdpa_dev;
2415         if (vdpa_dev && vdpa_dev->ops->migration_done)
2416                 vdpa_dev->ops->migration_done(dev->vid);
2417
2418         return RTE_VHOST_MSG_RESULT_OK;
2419 }
2420
2421 static int
2422 vhost_user_net_set_mtu(struct virtio_net **pdev,
2423                         struct vhu_msg_context *ctx,
2424                         int main_fd __rte_unused)
2425 {
2426         struct virtio_net *dev = *pdev;
2427
2428         if (validate_msg_fds(dev, ctx, 0) != 0)
2429                 return RTE_VHOST_MSG_RESULT_ERR;
2430
2431         if (ctx->msg.payload.u64 < VIRTIO_MIN_MTU ||
2432                         ctx->msg.payload.u64 > VIRTIO_MAX_MTU) {
2433                 VHOST_LOG_CONFIG(ERR, "(%s) invalid MTU size (%"PRIu64")\n",
2434                                 dev->ifname, ctx->msg.payload.u64);
2435
2436                 return RTE_VHOST_MSG_RESULT_ERR;
2437         }
2438
2439         dev->mtu = ctx->msg.payload.u64;
2440
2441         return RTE_VHOST_MSG_RESULT_OK;
2442 }
2443
2444 static int
2445 vhost_user_set_req_fd(struct virtio_net **pdev,
2446                         struct vhu_msg_context *ctx,
2447                         int main_fd __rte_unused)
2448 {
2449         struct virtio_net *dev = *pdev;
2450         int fd = ctx->fds[0];
2451
2452         if (validate_msg_fds(dev, ctx, 1) != 0)
2453                 return RTE_VHOST_MSG_RESULT_ERR;
2454
2455         if (fd < 0) {
2456                 VHOST_LOG_CONFIG(ERR, "(%s) invalid file descriptor for slave channel (%d)\n",
2457                                 dev->ifname, fd);
2458                 return RTE_VHOST_MSG_RESULT_ERR;
2459         }
2460
2461         if (dev->slave_req_fd >= 0)
2462                 close(dev->slave_req_fd);
2463
2464         dev->slave_req_fd = fd;
2465
2466         return RTE_VHOST_MSG_RESULT_OK;
2467 }
2468
2469 static int
2470 is_vring_iotlb_split(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2471 {
2472         struct vhost_vring_addr *ra;
2473         uint64_t start, end, len;
2474
2475         start = imsg->iova;
2476         end = start + imsg->size;
2477
2478         ra = &vq->ring_addrs;
2479         len = sizeof(struct vring_desc) * vq->size;
2480         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2481                 return 1;
2482
2483         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
2484         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2485                 return 1;
2486
2487         len = sizeof(struct vring_used) +
2488                sizeof(struct vring_used_elem) * vq->size;
2489         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2490                 return 1;
2491
2492         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2493                 len = sizeof(uint64_t);
2494                 if (ra->log_guest_addr < end &&
2495                     (ra->log_guest_addr + len) > start)
2496                         return 1;
2497         }
2498
2499         return 0;
2500 }
2501
2502 static int
2503 is_vring_iotlb_packed(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2504 {
2505         struct vhost_vring_addr *ra;
2506         uint64_t start, end, len;
2507
2508         start = imsg->iova;
2509         end = start + imsg->size;
2510
2511         ra = &vq->ring_addrs;
2512         len = sizeof(struct vring_packed_desc) * vq->size;
2513         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2514                 return 1;
2515
2516         len = sizeof(struct vring_packed_desc_event);
2517         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2518                 return 1;
2519
2520         len = sizeof(struct vring_packed_desc_event);
2521         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2522                 return 1;
2523
2524         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2525                 len = sizeof(uint64_t);
2526                 if (ra->log_guest_addr < end &&
2527                     (ra->log_guest_addr + len) > start)
2528                         return 1;
2529         }
2530
2531         return 0;
2532 }
2533
2534 static int is_vring_iotlb(struct virtio_net *dev,
2535                           struct vhost_virtqueue *vq,
2536                           struct vhost_iotlb_msg *imsg)
2537 {
2538         if (vq_is_packed(dev))
2539                 return is_vring_iotlb_packed(vq, imsg);
2540         else
2541                 return is_vring_iotlb_split(vq, imsg);
2542 }
2543
2544 static int
2545 vhost_user_iotlb_msg(struct virtio_net **pdev,
2546                         struct vhu_msg_context *ctx,
2547                         int main_fd __rte_unused)
2548 {
2549         struct virtio_net *dev = *pdev;
2550         struct vhost_iotlb_msg *imsg = &ctx->msg.payload.iotlb;
2551         uint16_t i;
2552         uint64_t vva, len;
2553
2554         if (validate_msg_fds(dev, ctx, 0) != 0)
2555                 return RTE_VHOST_MSG_RESULT_ERR;
2556
2557         switch (imsg->type) {
2558         case VHOST_IOTLB_UPDATE:
2559                 len = imsg->size;
2560                 vva = qva_to_vva(dev, imsg->uaddr, &len);
2561                 if (!vva)
2562                         return RTE_VHOST_MSG_RESULT_ERR;
2563
2564                 for (i = 0; i < dev->nr_vring; i++) {
2565                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2566
2567                         if (!vq)
2568                                 continue;
2569
2570                         vhost_user_iotlb_cache_insert(dev, vq, imsg->iova, vva,
2571                                         len, imsg->perm);
2572
2573                         if (is_vring_iotlb(dev, vq, imsg)) {
2574                                 rte_spinlock_lock(&vq->access_lock);
2575                                 *pdev = dev = translate_ring_addresses(dev, i);
2576                                 rte_spinlock_unlock(&vq->access_lock);
2577                         }
2578                 }
2579                 break;
2580         case VHOST_IOTLB_INVALIDATE:
2581                 for (i = 0; i < dev->nr_vring; i++) {
2582                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2583
2584                         if (!vq)
2585                                 continue;
2586
2587                         vhost_user_iotlb_cache_remove(vq, imsg->iova,
2588                                         imsg->size);
2589
2590                         if (is_vring_iotlb(dev, vq, imsg)) {
2591                                 rte_spinlock_lock(&vq->access_lock);
2592                                 vring_invalidate(dev, vq);
2593                                 rte_spinlock_unlock(&vq->access_lock);
2594                         }
2595                 }
2596                 break;
2597         default:
2598                 VHOST_LOG_CONFIG(ERR, "(%s) invalid IOTLB message type (%d)\n",
2599                                 dev->ifname, imsg->type);
2600                 return RTE_VHOST_MSG_RESULT_ERR;
2601         }
2602
2603         return RTE_VHOST_MSG_RESULT_OK;
2604 }
2605
2606 static int
2607 vhost_user_set_postcopy_advise(struct virtio_net **pdev,
2608                         struct vhu_msg_context *ctx,
2609                         int main_fd __rte_unused)
2610 {
2611         struct virtio_net *dev = *pdev;
2612 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
2613         struct uffdio_api api_struct;
2614
2615         if (validate_msg_fds(dev, ctx, 0) != 0)
2616                 return RTE_VHOST_MSG_RESULT_ERR;
2617
2618         dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
2619
2620         if (dev->postcopy_ufd == -1) {
2621                 VHOST_LOG_CONFIG(ERR, "(%s) userfaultfd not available: %s\n",
2622                         dev->ifname, strerror(errno));
2623                 return RTE_VHOST_MSG_RESULT_ERR;
2624         }
2625         api_struct.api = UFFD_API;
2626         api_struct.features = 0;
2627         if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
2628                 VHOST_LOG_CONFIG(ERR, "(%s) UFFDIO_API ioctl failure: %s\n",
2629                         dev->ifname, strerror(errno));
2630                 close(dev->postcopy_ufd);
2631                 dev->postcopy_ufd = -1;
2632                 return RTE_VHOST_MSG_RESULT_ERR;
2633         }
2634         ctx->fds[0] = dev->postcopy_ufd;
2635         ctx->fd_num = 1;
2636
2637         return RTE_VHOST_MSG_RESULT_REPLY;
2638 #else
2639         dev->postcopy_ufd = -1;
2640         ctx->fd_num = 0;
2641
2642         return RTE_VHOST_MSG_RESULT_ERR;
2643 #endif
2644 }
2645
2646 static int
2647 vhost_user_set_postcopy_listen(struct virtio_net **pdev,
2648                         struct vhu_msg_context *ctx __rte_unused,
2649                         int main_fd __rte_unused)
2650 {
2651         struct virtio_net *dev = *pdev;
2652
2653         if (validate_msg_fds(dev, ctx, 0) != 0)
2654                 return RTE_VHOST_MSG_RESULT_ERR;
2655
2656         if (dev->mem && dev->mem->nregions) {
2657                 VHOST_LOG_CONFIG(ERR, "(%s) regions already registered at postcopy-listen\n",
2658                                 dev->ifname);
2659                 return RTE_VHOST_MSG_RESULT_ERR;
2660         }
2661         dev->postcopy_listening = 1;
2662
2663         return RTE_VHOST_MSG_RESULT_OK;
2664 }
2665
2666 static int
2667 vhost_user_postcopy_end(struct virtio_net **pdev,
2668                         struct vhu_msg_context *ctx,
2669                         int main_fd __rte_unused)
2670 {
2671         struct virtio_net *dev = *pdev;
2672
2673         if (validate_msg_fds(dev, ctx, 0) != 0)
2674                 return RTE_VHOST_MSG_RESULT_ERR;
2675
2676         dev->postcopy_listening = 0;
2677         if (dev->postcopy_ufd >= 0) {
2678                 close(dev->postcopy_ufd);
2679                 dev->postcopy_ufd = -1;
2680         }
2681
2682         ctx->msg.payload.u64 = 0;
2683         ctx->msg.size = sizeof(ctx->msg.payload.u64);
2684         ctx->fd_num = 0;
2685
2686         return RTE_VHOST_MSG_RESULT_REPLY;
2687 }
2688
2689 static int
2690 vhost_user_get_status(struct virtio_net **pdev,
2691                       struct vhu_msg_context *ctx,
2692                       int main_fd __rte_unused)
2693 {
2694         struct virtio_net *dev = *pdev;
2695
2696         if (validate_msg_fds(dev, ctx, 0) != 0)
2697                 return RTE_VHOST_MSG_RESULT_ERR;
2698
2699         ctx->msg.payload.u64 = dev->status;
2700         ctx->msg.size = sizeof(ctx->msg.payload.u64);
2701         ctx->fd_num = 0;
2702
2703         return RTE_VHOST_MSG_RESULT_REPLY;
2704 }
2705
2706 static int
2707 vhost_user_set_status(struct virtio_net **pdev,
2708                         struct vhu_msg_context *ctx,
2709                         int main_fd __rte_unused)
2710 {
2711         struct virtio_net *dev = *pdev;
2712
2713         if (validate_msg_fds(dev, ctx, 0) != 0)
2714                 return RTE_VHOST_MSG_RESULT_ERR;
2715
2716         /* As per Virtio specification, the device status is 8bits long */
2717         if (ctx->msg.payload.u64 > UINT8_MAX) {
2718                 VHOST_LOG_CONFIG(ERR, "(%s) invalid VHOST_USER_SET_STATUS payload 0x%" PRIx64 "\n",
2719                                 dev->ifname, ctx->msg.payload.u64);
2720                 return RTE_VHOST_MSG_RESULT_ERR;
2721         }
2722
2723         dev->status = ctx->msg.payload.u64;
2724
2725         if ((dev->status & VIRTIO_DEVICE_STATUS_FEATURES_OK) &&
2726             (dev->flags & VIRTIO_DEV_FEATURES_FAILED)) {
2727                 VHOST_LOG_CONFIG(ERR,
2728                                 "(%s) FEATURES_OK bit is set but feature negotiation failed\n",
2729                                 dev->ifname);
2730                 /*
2731                  * Clear the bit to let the driver know about the feature
2732                  * negotiation failure
2733                  */
2734                 dev->status &= ~VIRTIO_DEVICE_STATUS_FEATURES_OK;
2735         }
2736
2737         VHOST_LOG_CONFIG(INFO, "(%s) new device status(0x%08x):\n", dev->ifname,
2738                         dev->status);
2739         VHOST_LOG_CONFIG(INFO, "(%s)\t-RESET: %u\n", dev->ifname,
2740                         (dev->status == VIRTIO_DEVICE_STATUS_RESET));
2741         VHOST_LOG_CONFIG(INFO, "(%s)\t-ACKNOWLEDGE: %u\n", dev->ifname,
2742                         !!(dev->status & VIRTIO_DEVICE_STATUS_ACK));
2743         VHOST_LOG_CONFIG(INFO, "(%s)\t-DRIVER: %u\n", dev->ifname,
2744                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER));
2745         VHOST_LOG_CONFIG(INFO, "(%s)\t-FEATURES_OK: %u\n", dev->ifname,
2746                         !!(dev->status & VIRTIO_DEVICE_STATUS_FEATURES_OK));
2747         VHOST_LOG_CONFIG(INFO, "(%s)\t-DRIVER_OK: %u\n", dev->ifname,
2748                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK));
2749         VHOST_LOG_CONFIG(INFO, "(%s)\t-DEVICE_NEED_RESET: %u\n", dev->ifname,
2750                         !!(dev->status & VIRTIO_DEVICE_STATUS_DEV_NEED_RESET));
2751         VHOST_LOG_CONFIG(INFO, "(%s)\t-FAILED: %u\n", dev->ifname,
2752                         !!(dev->status & VIRTIO_DEVICE_STATUS_FAILED));
2753
2754         return RTE_VHOST_MSG_RESULT_OK;
2755 }
2756
2757 typedef int (*vhost_message_handler_t)(struct virtio_net **pdev,
2758                                         struct vhu_msg_context *ctx,
2759                                         int main_fd);
2760
2761 static vhost_message_handler_t vhost_message_handlers[VHOST_USER_MAX] = {
2762         [VHOST_USER_NONE] = NULL,
2763         [VHOST_USER_GET_FEATURES] = vhost_user_get_features,
2764         [VHOST_USER_SET_FEATURES] = vhost_user_set_features,
2765         [VHOST_USER_SET_OWNER] = vhost_user_set_owner,
2766         [VHOST_USER_RESET_OWNER] = vhost_user_reset_owner,
2767         [VHOST_USER_SET_MEM_TABLE] = vhost_user_set_mem_table,
2768         [VHOST_USER_SET_LOG_BASE] = vhost_user_set_log_base,
2769         [VHOST_USER_SET_LOG_FD] = vhost_user_set_log_fd,
2770         [VHOST_USER_SET_VRING_NUM] = vhost_user_set_vring_num,
2771         [VHOST_USER_SET_VRING_ADDR] = vhost_user_set_vring_addr,
2772         [VHOST_USER_SET_VRING_BASE] = vhost_user_set_vring_base,
2773         [VHOST_USER_GET_VRING_BASE] = vhost_user_get_vring_base,
2774         [VHOST_USER_SET_VRING_KICK] = vhost_user_set_vring_kick,
2775         [VHOST_USER_SET_VRING_CALL] = vhost_user_set_vring_call,
2776         [VHOST_USER_SET_VRING_ERR] = vhost_user_set_vring_err,
2777         [VHOST_USER_GET_PROTOCOL_FEATURES] = vhost_user_get_protocol_features,
2778         [VHOST_USER_SET_PROTOCOL_FEATURES] = vhost_user_set_protocol_features,
2779         [VHOST_USER_GET_QUEUE_NUM] = vhost_user_get_queue_num,
2780         [VHOST_USER_SET_VRING_ENABLE] = vhost_user_set_vring_enable,
2781         [VHOST_USER_SEND_RARP] = vhost_user_send_rarp,
2782         [VHOST_USER_NET_SET_MTU] = vhost_user_net_set_mtu,
2783         [VHOST_USER_SET_SLAVE_REQ_FD] = vhost_user_set_req_fd,
2784         [VHOST_USER_IOTLB_MSG] = vhost_user_iotlb_msg,
2785         [VHOST_USER_POSTCOPY_ADVISE] = vhost_user_set_postcopy_advise,
2786         [VHOST_USER_POSTCOPY_LISTEN] = vhost_user_set_postcopy_listen,
2787         [VHOST_USER_POSTCOPY_END] = vhost_user_postcopy_end,
2788         [VHOST_USER_GET_INFLIGHT_FD] = vhost_user_get_inflight_fd,
2789         [VHOST_USER_SET_INFLIGHT_FD] = vhost_user_set_inflight_fd,
2790         [VHOST_USER_SET_STATUS] = vhost_user_set_status,
2791         [VHOST_USER_GET_STATUS] = vhost_user_get_status,
2792 };
2793
2794 /* return bytes# of read on success or negative val on failure. */
2795 static int
2796 read_vhost_message(struct virtio_net *dev, int sockfd, struct  vhu_msg_context *ctx)
2797 {
2798         int ret;
2799
2800         ret = read_fd_message(dev->ifname, sockfd, (char *)&ctx->msg, VHOST_USER_HDR_SIZE,
2801                 ctx->fds, VHOST_MEMORY_MAX_NREGIONS, &ctx->fd_num);
2802         if (ret <= 0) {
2803                 return ret;
2804         } else if (ret != VHOST_USER_HDR_SIZE) {
2805                 VHOST_LOG_CONFIG(ERR, "(%s) Unexpected header size read\n", dev->ifname);
2806                 close_msg_fds(ctx);
2807                 return -1;
2808         }
2809
2810         if (ctx->msg.size) {
2811                 if (ctx->msg.size > sizeof(ctx->msg.payload)) {
2812                         VHOST_LOG_CONFIG(ERR, "(%s) invalid msg size: %d\n",
2813                                         dev->ifname, ctx->msg.size);
2814                         return -1;
2815                 }
2816                 ret = read(sockfd, &ctx->msg.payload, ctx->msg.size);
2817                 if (ret <= 0)
2818                         return ret;
2819                 if (ret != (int)ctx->msg.size) {
2820                         VHOST_LOG_CONFIG(ERR, "(%s) read control message failed\n", dev->ifname);
2821                         return -1;
2822                 }
2823         }
2824
2825         return ret;
2826 }
2827
2828 static int
2829 send_vhost_message(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx)
2830 {
2831         if (!ctx)
2832                 return 0;
2833
2834         return send_fd_message(dev->ifname, sockfd, (char *)&ctx->msg,
2835                 VHOST_USER_HDR_SIZE + ctx->msg.size, ctx->fds, ctx->fd_num);
2836 }
2837
2838 static int
2839 send_vhost_reply(struct virtio_net *dev, int sockfd, struct vhu_msg_context *ctx)
2840 {
2841         if (!ctx)
2842                 return 0;
2843
2844         ctx->msg.flags &= ~VHOST_USER_VERSION_MASK;
2845         ctx->msg.flags &= ~VHOST_USER_NEED_REPLY;
2846         ctx->msg.flags |= VHOST_USER_VERSION;
2847         ctx->msg.flags |= VHOST_USER_REPLY_MASK;
2848
2849         return send_vhost_message(dev, sockfd, ctx);
2850 }
2851
2852 static int
2853 send_vhost_slave_message(struct virtio_net *dev,
2854                 struct vhu_msg_context *ctx)
2855 {
2856         int ret;
2857
2858         if (ctx->msg.flags & VHOST_USER_NEED_REPLY)
2859                 rte_spinlock_lock(&dev->slave_req_lock);
2860
2861         ret = send_vhost_message(dev, dev->slave_req_fd, ctx);
2862         if (ret < 0 && (ctx->msg.flags & VHOST_USER_NEED_REPLY))
2863                 rte_spinlock_unlock(&dev->slave_req_lock);
2864
2865         return ret;
2866 }
2867
2868 /*
2869  * Allocate a queue pair if it hasn't been allocated yet
2870  */
2871 static int
2872 vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev,
2873                         struct vhu_msg_context *ctx)
2874 {
2875         uint32_t vring_idx;
2876
2877         switch (ctx->msg.request.master) {
2878         case VHOST_USER_SET_VRING_KICK:
2879         case VHOST_USER_SET_VRING_CALL:
2880         case VHOST_USER_SET_VRING_ERR:
2881                 vring_idx = ctx->msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
2882                 break;
2883         case VHOST_USER_SET_VRING_NUM:
2884         case VHOST_USER_SET_VRING_BASE:
2885         case VHOST_USER_GET_VRING_BASE:
2886         case VHOST_USER_SET_VRING_ENABLE:
2887                 vring_idx = ctx->msg.payload.state.index;
2888                 break;
2889         case VHOST_USER_SET_VRING_ADDR:
2890                 vring_idx = ctx->msg.payload.addr.index;
2891                 break;
2892         case VHOST_USER_SET_INFLIGHT_FD:
2893                 vring_idx = ctx->msg.payload.inflight.num_queues - 1;
2894                 break;
2895         default:
2896                 return 0;
2897         }
2898
2899         if (vring_idx >= VHOST_MAX_VRING) {
2900                 VHOST_LOG_CONFIG(ERR, "(%s) invalid vring index: %u\n", dev->ifname, vring_idx);
2901                 return -1;
2902         }
2903
2904         if (dev->virtqueue[vring_idx])
2905                 return 0;
2906
2907         return alloc_vring_queue(dev, vring_idx);
2908 }
2909
2910 static void
2911 vhost_user_lock_all_queue_pairs(struct virtio_net *dev)
2912 {
2913         unsigned int i = 0;
2914         unsigned int vq_num = 0;
2915
2916         while (vq_num < dev->nr_vring) {
2917                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2918
2919                 if (vq) {
2920                         rte_spinlock_lock(&vq->access_lock);
2921                         vq_num++;
2922                 }
2923                 i++;
2924         }
2925 }
2926
2927 static void
2928 vhost_user_unlock_all_queue_pairs(struct virtio_net *dev)
2929 {
2930         unsigned int i = 0;
2931         unsigned int vq_num = 0;
2932
2933         while (vq_num < dev->nr_vring) {
2934                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2935
2936                 if (vq) {
2937                         rte_spinlock_unlock(&vq->access_lock);
2938                         vq_num++;
2939                 }
2940                 i++;
2941         }
2942 }
2943
2944 int
2945 vhost_user_msg_handler(int vid, int fd)
2946 {
2947         struct virtio_net *dev;
2948         struct vhu_msg_context ctx;
2949         struct rte_vdpa_device *vdpa_dev;
2950         int ret;
2951         int unlock_required = 0;
2952         bool handled;
2953         int request;
2954         uint32_t i;
2955
2956         dev = get_device(vid);
2957         if (dev == NULL)
2958                 return -1;
2959
2960         if (!dev->notify_ops) {
2961                 dev->notify_ops = vhost_driver_callback_get(dev->ifname);
2962                 if (!dev->notify_ops) {
2963                         VHOST_LOG_CONFIG(ERR, "(%s) failed to get callback ops for driver\n",
2964                                 dev->ifname);
2965                         return -1;
2966                 }
2967         }
2968
2969         ret = read_vhost_message(dev, fd, &ctx);
2970         if (ret <= 0) {
2971                 if (ret < 0)
2972                         VHOST_LOG_CONFIG(ERR, "(%s) vhost read message failed\n", dev->ifname);
2973                 else
2974                         VHOST_LOG_CONFIG(INFO, "(%s) vhost peer closed\n", dev->ifname);
2975
2976                 return -1;
2977         }
2978
2979         ret = 0;
2980         request = ctx.msg.request.master;
2981         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX &&
2982                         vhost_message_str[request]) {
2983                 if (request != VHOST_USER_IOTLB_MSG)
2984                         VHOST_LOG_CONFIG(INFO, "(%s) read message %s\n",
2985                                 dev->ifname, vhost_message_str[request]);
2986                 else
2987                         VHOST_LOG_CONFIG(DEBUG, "(%s) read message %s\n",
2988                                 dev->ifname, vhost_message_str[request]);
2989         } else {
2990                 VHOST_LOG_CONFIG(DEBUG, "(%s) external request %d\n", dev->ifname, request);
2991         }
2992
2993         ret = vhost_user_check_and_alloc_queue_pair(dev, &ctx);
2994         if (ret < 0) {
2995                 VHOST_LOG_CONFIG(ERR, "(%s) failed to alloc queue\n", dev->ifname);
2996                 return -1;
2997         }
2998
2999         /*
3000          * Note: we don't lock all queues on VHOST_USER_GET_VRING_BASE
3001          * and VHOST_USER_RESET_OWNER, since it is sent when virtio stops
3002          * and device is destroyed. destroy_device waits for queues to be
3003          * inactive, so it is safe. Otherwise taking the access_lock
3004          * would cause a dead lock.
3005          */
3006         switch (request) {
3007         case VHOST_USER_SET_FEATURES:
3008         case VHOST_USER_SET_PROTOCOL_FEATURES:
3009         case VHOST_USER_SET_OWNER:
3010         case VHOST_USER_SET_MEM_TABLE:
3011         case VHOST_USER_SET_LOG_BASE:
3012         case VHOST_USER_SET_LOG_FD:
3013         case VHOST_USER_SET_VRING_NUM:
3014         case VHOST_USER_SET_VRING_ADDR:
3015         case VHOST_USER_SET_VRING_BASE:
3016         case VHOST_USER_SET_VRING_KICK:
3017         case VHOST_USER_SET_VRING_CALL:
3018         case VHOST_USER_SET_VRING_ERR:
3019         case VHOST_USER_SET_VRING_ENABLE:
3020         case VHOST_USER_SEND_RARP:
3021         case VHOST_USER_NET_SET_MTU:
3022         case VHOST_USER_SET_SLAVE_REQ_FD:
3023                 if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
3024                         vhost_user_lock_all_queue_pairs(dev);
3025                         unlock_required = 1;
3026                 }
3027                 break;
3028         default:
3029                 break;
3030
3031         }
3032
3033         handled = false;
3034         if (dev->extern_ops.pre_msg_handle) {
3035                 RTE_BUILD_BUG_ON(offsetof(struct vhu_msg_context, msg) != 0);
3036                 ret = (*dev->extern_ops.pre_msg_handle)(dev->vid, &ctx);
3037                 switch (ret) {
3038                 case RTE_VHOST_MSG_RESULT_REPLY:
3039                         send_vhost_reply(dev, fd, &ctx);
3040                         /* Fall-through */
3041                 case RTE_VHOST_MSG_RESULT_ERR:
3042                 case RTE_VHOST_MSG_RESULT_OK:
3043                         handled = true;
3044                         goto skip_to_post_handle;
3045                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
3046                 default:
3047                         break;
3048                 }
3049         }
3050
3051         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX) {
3052                 if (!vhost_message_handlers[request])
3053                         goto skip_to_post_handle;
3054                 ret = vhost_message_handlers[request](&dev, &ctx, fd);
3055
3056                 switch (ret) {
3057                 case RTE_VHOST_MSG_RESULT_ERR:
3058                         VHOST_LOG_CONFIG(ERR, "(%s) processing %s failed.\n",
3059                                         dev->ifname, vhost_message_str[request]);
3060                         handled = true;
3061                         break;
3062                 case RTE_VHOST_MSG_RESULT_OK:
3063                         VHOST_LOG_CONFIG(DEBUG, "(%s) processing %s succeeded.\n",
3064                                         dev->ifname, vhost_message_str[request]);
3065                         handled = true;
3066                         break;
3067                 case RTE_VHOST_MSG_RESULT_REPLY:
3068                         VHOST_LOG_CONFIG(DEBUG, "(%s) processing %s succeeded and needs reply.\n",
3069                                         dev->ifname, vhost_message_str[request]);
3070                         send_vhost_reply(dev, fd, &ctx);
3071                         handled = true;
3072                         break;
3073                 default:
3074                         break;
3075                 }
3076         }
3077
3078 skip_to_post_handle:
3079         if (ret != RTE_VHOST_MSG_RESULT_ERR &&
3080                         dev->extern_ops.post_msg_handle) {
3081                 RTE_BUILD_BUG_ON(offsetof(struct vhu_msg_context, msg) != 0);
3082                 ret = (*dev->extern_ops.post_msg_handle)(dev->vid, &ctx);
3083                 switch (ret) {
3084                 case RTE_VHOST_MSG_RESULT_REPLY:
3085                         send_vhost_reply(dev, fd, &ctx);
3086                         /* Fall-through */
3087                 case RTE_VHOST_MSG_RESULT_ERR:
3088                 case RTE_VHOST_MSG_RESULT_OK:
3089                         handled = true;
3090                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
3091                 default:
3092                         break;
3093                 }
3094         }
3095
3096         /* If message was not handled at this stage, treat it as an error */
3097         if (!handled) {
3098                 VHOST_LOG_CONFIG(ERR, "(%s) vhost message (req: %d) was not handled.\n",
3099                                 dev->ifname, request);
3100                 close_msg_fds(&ctx);
3101                 ret = RTE_VHOST_MSG_RESULT_ERR;
3102         }
3103
3104         /*
3105          * If the request required a reply that was already sent,
3106          * this optional reply-ack won't be sent as the
3107          * VHOST_USER_NEED_REPLY was cleared in send_vhost_reply().
3108          */
3109         if (ctx.msg.flags & VHOST_USER_NEED_REPLY) {
3110                 ctx.msg.payload.u64 = ret == RTE_VHOST_MSG_RESULT_ERR;
3111                 ctx.msg.size = sizeof(ctx.msg.payload.u64);
3112                 ctx.fd_num = 0;
3113                 send_vhost_reply(dev, fd, &ctx);
3114         } else if (ret == RTE_VHOST_MSG_RESULT_ERR) {
3115                 VHOST_LOG_CONFIG(ERR, "(%s) vhost message handling failed.\n", dev->ifname);
3116                 return -1;
3117         }
3118
3119         for (i = 0; i < dev->nr_vring; i++) {
3120                 struct vhost_virtqueue *vq = dev->virtqueue[i];
3121                 bool cur_ready = vq_is_ready(dev, vq);
3122
3123                 if (cur_ready != (vq && vq->ready)) {
3124                         vq->ready = cur_ready;
3125                         vhost_user_notify_queue_state(dev, i, cur_ready);
3126                 }
3127         }
3128
3129         if (unlock_required)
3130                 vhost_user_unlock_all_queue_pairs(dev);
3131
3132         if (!virtio_is_ready(dev))
3133                 goto out;
3134
3135         /*
3136          * Virtio is now ready. If not done already, it is time
3137          * to notify the application it can process the rings and
3138          * configure the vDPA device if present.
3139          */
3140
3141         if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
3142                 if (dev->notify_ops->new_device(dev->vid) == 0)
3143                         dev->flags |= VIRTIO_DEV_RUNNING;
3144         }
3145
3146         vdpa_dev = dev->vdpa_dev;
3147         if (!vdpa_dev)
3148                 goto out;
3149
3150         if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
3151                 if (vdpa_dev->ops->dev_conf(dev->vid))
3152                         VHOST_LOG_CONFIG(ERR, "(%s) failed to configure vDPA device\n",
3153                                         dev->ifname);
3154                 else
3155                         dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED;
3156         }
3157
3158 out:
3159         return 0;
3160 }
3161
3162 static int process_slave_message_reply(struct virtio_net *dev,
3163                                        const struct vhu_msg_context *ctx)
3164 {
3165         struct vhu_msg_context msg_reply;
3166         int ret;
3167
3168         if ((ctx->msg.flags & VHOST_USER_NEED_REPLY) == 0)
3169                 return 0;
3170
3171         ret = read_vhost_message(dev, dev->slave_req_fd, &msg_reply);
3172         if (ret <= 0) {
3173                 if (ret < 0)
3174                         VHOST_LOG_CONFIG(ERR, "(%s) vhost read slave message reply failed\n",
3175                                         dev->ifname);
3176                 else
3177                         VHOST_LOG_CONFIG(INFO, "(%s) vhost peer closed\n", dev->ifname);
3178                 ret = -1;
3179                 goto out;
3180         }
3181
3182         ret = 0;
3183         if (msg_reply.msg.request.slave != ctx->msg.request.slave) {
3184                 VHOST_LOG_CONFIG(ERR, "(%s) received unexpected msg type (%u), expected %u\n",
3185                                 dev->ifname, msg_reply.msg.request.slave, ctx->msg.request.slave);
3186                 ret = -1;
3187                 goto out;
3188         }
3189
3190         ret = msg_reply.msg.payload.u64 ? -1 : 0;
3191
3192 out:
3193         rte_spinlock_unlock(&dev->slave_req_lock);
3194         return ret;
3195 }
3196
3197 int
3198 vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
3199 {
3200         int ret;
3201         struct vhu_msg_context ctx = {
3202                 .msg = {
3203                         .request.slave = VHOST_USER_SLAVE_IOTLB_MSG,
3204                         .flags = VHOST_USER_VERSION,
3205                         .size = sizeof(ctx.msg.payload.iotlb),
3206                         .payload.iotlb = {
3207                                 .iova = iova,
3208                                 .perm = perm,
3209                                 .type = VHOST_IOTLB_MISS,
3210                         },
3211                 },
3212         };
3213
3214         ret = send_vhost_message(dev, dev->slave_req_fd, &ctx);
3215         if (ret < 0) {
3216                 VHOST_LOG_CONFIG(ERR, "(%s) failed to send IOTLB miss message (%d)\n",
3217                                 dev->ifname, ret);
3218                 return ret;
3219         }
3220
3221         return 0;
3222 }
3223
3224 static int
3225 vhost_user_slave_config_change(struct virtio_net *dev, bool need_reply)
3226 {
3227         int ret;
3228         struct vhu_msg_context ctx = {
3229                 .msg = {
3230                         .request.slave = VHOST_USER_SLAVE_CONFIG_CHANGE_MSG,
3231                         .flags = VHOST_USER_VERSION,
3232                         .size = 0,
3233                 }
3234         };
3235
3236         if (need_reply)
3237                 ctx.msg.flags |= VHOST_USER_NEED_REPLY;
3238
3239         ret = send_vhost_slave_message(dev, &ctx);
3240         if (ret < 0) {
3241                 VHOST_LOG_CONFIG(ERR, "(%s) failed to send config change (%d)\n",
3242                                 dev->ifname, ret);
3243                 return ret;
3244         }
3245
3246         return process_slave_message_reply(dev, &ctx);
3247 }
3248
3249 int
3250 rte_vhost_slave_config_change(int vid, bool need_reply)
3251 {
3252         struct virtio_net *dev;
3253
3254         dev = get_device(vid);
3255         if (!dev)
3256                 return -ENODEV;
3257
3258         return vhost_user_slave_config_change(dev, need_reply);
3259 }
3260
3261 static int vhost_user_slave_set_vring_host_notifier(struct virtio_net *dev,
3262                                                     int index, int fd,
3263                                                     uint64_t offset,
3264                                                     uint64_t size)
3265 {
3266         int ret;
3267         struct vhu_msg_context ctx = {
3268                 .msg = {
3269                         .request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
3270                         .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY,
3271                         .size = sizeof(ctx.msg.payload.area),
3272                         .payload.area = {
3273                                 .u64 = index & VHOST_USER_VRING_IDX_MASK,
3274                                 .size = size,
3275                                 .offset = offset,
3276                         },
3277                 },
3278         };
3279
3280         if (fd < 0)
3281                 ctx.msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
3282         else {
3283                 ctx.fds[0] = fd;
3284                 ctx.fd_num = 1;
3285         }
3286
3287         ret = send_vhost_slave_message(dev, &ctx);
3288         if (ret < 0) {
3289                 VHOST_LOG_CONFIG(ERR, "(%s) failed to set host notifier (%d)\n",
3290                                 dev->ifname, ret);
3291                 return ret;
3292         }
3293
3294         return process_slave_message_reply(dev, &ctx);
3295 }
3296
3297 int rte_vhost_host_notifier_ctrl(int vid, uint16_t qid, bool enable)
3298 {
3299         struct virtio_net *dev;
3300         struct rte_vdpa_device *vdpa_dev;
3301         int vfio_device_fd, ret = 0;
3302         uint64_t offset, size;
3303         unsigned int i, q_start, q_last;
3304
3305         dev = get_device(vid);
3306         if (!dev)
3307                 return -ENODEV;
3308
3309         vdpa_dev = dev->vdpa_dev;
3310         if (vdpa_dev == NULL)
3311                 return -ENODEV;
3312
3313         if (!(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
3314             !(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) ||
3315             !(dev->protocol_features &
3316                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) ||
3317             !(dev->protocol_features &
3318                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD)) ||
3319             !(dev->protocol_features &
3320                         (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER)))
3321                 return -ENOTSUP;
3322
3323         if (qid == RTE_VHOST_QUEUE_ALL) {
3324                 q_start = 0;
3325                 q_last = dev->nr_vring - 1;
3326         } else {
3327                 if (qid >= dev->nr_vring)
3328                         return -EINVAL;
3329                 q_start = qid;
3330                 q_last = qid;
3331         }
3332
3333         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_vfio_device_fd, -ENOTSUP);
3334         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_notify_area, -ENOTSUP);
3335
3336         vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid);
3337         if (vfio_device_fd < 0)
3338                 return -ENOTSUP;
3339
3340         if (enable) {
3341                 for (i = q_start; i <= q_last; i++) {
3342                         if (vdpa_dev->ops->get_notify_area(vid, i, &offset,
3343                                         &size) < 0) {
3344                                 ret = -ENOTSUP;
3345                                 goto disable;
3346                         }
3347
3348                         if (vhost_user_slave_set_vring_host_notifier(dev, i,
3349                                         vfio_device_fd, offset, size) < 0) {
3350                                 ret = -EFAULT;
3351                                 goto disable;
3352                         }
3353                 }
3354         } else {
3355 disable:
3356                 for (i = q_start; i <= q_last; i++) {
3357                         vhost_user_slave_set_vring_host_notifier(dev, i, -1,
3358                                         0, 0);
3359                 }
3360         }
3361
3362         return ret;
3363 }