lib/librte_vhost/vhost_user.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 /* Security model
   6  * --------------
   7  * The vhost-user protocol connection is an external interface, so it must be
   8  * robust against invalid inputs.
   9  *
  10  * This is important because the vhost-user master is only one step removed
  11  * from the guest.  Malicious guests that have escaped will then launch further
  12  * attacks from the vhost-user master.
  13  *
  14  * Even in deployments where guests are trusted, a bug in the vhost-user master
  15  * can still cause invalid messages to be sent.  Such messages must not
  16  * compromise the stability of the DPDK application by causing crashes, memory
  17  * corruption, or other problematic behavior.
  18  *
  19  * Do not assume received VhostUserMsg fields contain sensible values!
  20  */
  21
  22 #include <stdint.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28 #include <sys/ioctl.h>
  29 #include <sys/mman.h>
  30 #include <sys/types.h>
  31 #include <sys/stat.h>
  32 #include <sys/syscall.h>
  33 #include <assert.h>
  34 #ifdef RTE_LIBRTE_VHOST_NUMA
  35 #include <numaif.h>
  36 #endif
  37 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
  38 #include <linux/userfaultfd.h>
  39 #endif
  40 #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
  41 #include <linux/memfd.h>
  42 #define MEMFD_SUPPORTED
  43 #endif
  44
  45 #include <rte_common.h>
  46 #include <rte_malloc.h>
  47 #include <rte_log.h>
  48
  49 #include "iotlb.h"
  50 #include "vhost.h"
  51 #include "vhost_user.h"
  52
  53 #define VIRTIO_MIN_MTU 68
  54 #define VIRTIO_MAX_MTU 65535
  55
  56 #define INFLIGHT_ALIGNMENT      64
  57 #define INFLIGHT_VERSION        0x1
  58
  59 static const char *vhost_message_str[VHOST_USER_MAX] = {
  60         [VHOST_USER_NONE] = "VHOST_USER_NONE",
  61         [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
  62         [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
  63         [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
  64         [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
  65         [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
  66         [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
  67         [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
  68         [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
  69         [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
  70         [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
  71         [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
  72         [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
  73         [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
  74         [VHOST_USER_SET_VRING_ERR]  = "VHOST_USER_SET_VRING_ERR",
  75         [VHOST_USER_GET_PROTOCOL_FEATURES]  = "VHOST_USER_GET_PROTOCOL_FEATURES",
  76         [VHOST_USER_SET_PROTOCOL_FEATURES]  = "VHOST_USER_SET_PROTOCOL_FEATURES",
  77         [VHOST_USER_GET_QUEUE_NUM]  = "VHOST_USER_GET_QUEUE_NUM",
  78         [VHOST_USER_SET_VRING_ENABLE]  = "VHOST_USER_SET_VRING_ENABLE",
  79         [VHOST_USER_SEND_RARP]  = "VHOST_USER_SEND_RARP",
  80         [VHOST_USER_NET_SET_MTU]  = "VHOST_USER_NET_SET_MTU",
  81         [VHOST_USER_SET_SLAVE_REQ_FD]  = "VHOST_USER_SET_SLAVE_REQ_FD",
  82         [VHOST_USER_IOTLB_MSG]  = "VHOST_USER_IOTLB_MSG",
  83         [VHOST_USER_CRYPTO_CREATE_SESS] = "VHOST_USER_CRYPTO_CREATE_SESS",
  84         [VHOST_USER_CRYPTO_CLOSE_SESS] = "VHOST_USER_CRYPTO_CLOSE_SESS",
  85         [VHOST_USER_POSTCOPY_ADVISE]  = "VHOST_USER_POSTCOPY_ADVISE",
  86         [VHOST_USER_POSTCOPY_LISTEN]  = "VHOST_USER_POSTCOPY_LISTEN",
  87         [VHOST_USER_POSTCOPY_END]  = "VHOST_USER_POSTCOPY_END",
  88         [VHOST_USER_GET_INFLIGHT_FD] = "VHOST_USER_GET_INFLIGHT_FD",
  89         [VHOST_USER_SET_INFLIGHT_FD] = "VHOST_USER_SET_INFLIGHT_FD",
  90         [VHOST_USER_SET_STATUS] = "VHOST_USER_SET_STATUS",
  91         [VHOST_USER_GET_STATUS] = "VHOST_USER_GET_STATUS",
  92 };
  93
  94 static int send_vhost_reply(int sockfd, struct VhostUserMsg *msg);
  95 static int read_vhost_message(int sockfd, struct VhostUserMsg *msg);
  96
  97 static void
  98 close_msg_fds(struct VhostUserMsg *msg)
  99 {
 100         int i;
 101
 102         for (i = 0; i < msg->fd_num; i++)
 103                 close(msg->fds[i]);
 104 }
 105
 106 /*
 107  * Ensure the expected number of FDs is received,
 108  * close all FDs and return an error if this is not the case.
 109  */
 110 static int
 111 validate_msg_fds(struct VhostUserMsg *msg, int expected_fds)
 112 {
 113         if (msg->fd_num == expected_fds)
 114                 return 0;
 115
 116         VHOST_LOG_CONFIG(ERR,
 117                 " Expect %d FDs for request %s, received %d\n",
 118                 expected_fds,
 119                 vhost_message_str[msg->request.master],
 120                 msg->fd_num);
 121
 122         close_msg_fds(msg);
 123
 124         return -1;
 125 }
 126
 127 static uint64_t
 128 get_blk_size(int fd)
 129 {
 130         struct stat stat;
 131         int ret;
 132
 133         ret = fstat(fd, &stat);
 134         return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
 135 }
 136
 137 static void
 138 free_mem_region(struct virtio_net *dev)
 139 {
 140         uint32_t i;
 141         struct rte_vhost_mem_region *reg;
 142
 143         if (!dev || !dev->mem)
 144                 return;
 145
 146         for (i = 0; i < dev->mem->nregions; i++) {
 147                 reg = &dev->mem->regions[i];
 148                 if (reg->host_user_addr) {
 149                         munmap(reg->mmap_addr, reg->mmap_size);
 150                         close(reg->fd);
 151                 }
 152         }
 153 }
 154
 155 void
 156 vhost_backend_cleanup(struct virtio_net *dev)
 157 {
 158         if (dev->mem) {
 159                 free_mem_region(dev);
 160                 rte_free(dev->mem);
 161                 dev->mem = NULL;
 162         }
 163
 164         rte_free(dev->guest_pages);
 165         dev->guest_pages = NULL;
 166
 167         if (dev->log_addr) {
 168                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 169                 dev->log_addr = 0;
 170         }
 171
 172         if (dev->inflight_info) {
 173                 if (dev->inflight_info->addr) {
 174                         munmap(dev->inflight_info->addr,
 175                                dev->inflight_info->size);
 176                         dev->inflight_info->addr = NULL;
 177                 }
 178
 179                 if (dev->inflight_info->fd >= 0) {
 180                         close(dev->inflight_info->fd);
 181                         dev->inflight_info->fd = -1;
 182                 }
 183
 184                 free(dev->inflight_info);
 185                 dev->inflight_info = NULL;
 186         }
 187
 188         if (dev->slave_req_fd >= 0) {
 189                 close(dev->slave_req_fd);
 190                 dev->slave_req_fd = -1;
 191         }
 192
 193         if (dev->postcopy_ufd >= 0) {
 194                 close(dev->postcopy_ufd);
 195                 dev->postcopy_ufd = -1;
 196         }
 197
 198         dev->postcopy_listening = 0;
 199 }
 200
 201 static void
 202 vhost_user_notify_queue_state(struct virtio_net *dev, uint16_t index,
 203                               int enable)
 204 {
 205         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
 206         struct vhost_virtqueue *vq = dev->virtqueue[index];
 207
 208         /* Configure guest notifications on enable */
 209         if (enable && vq->notif_enable != VIRTIO_UNINITIALIZED_NOTIF)
 210                 vhost_enable_guest_notification(dev, vq, vq->notif_enable);
 211
 212         if (vdpa_dev && vdpa_dev->ops->set_vring_state)
 213                 vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
 214
 215         if (dev->notify_ops->vring_state_changed)
 216                 dev->notify_ops->vring_state_changed(dev->vid,
 217                                 index, enable);
 218 }
 219
 220 /*
 221  * This function just returns success at the moment unless
 222  * the device hasn't been initialised.
 223  */
 224 static int
 225 vhost_user_set_owner(struct virtio_net **pdev __rte_unused,
 226                         struct VhostUserMsg *msg,
 227                         int main_fd __rte_unused)
 228 {
 229         if (validate_msg_fds(msg, 0) != 0)
 230                 return RTE_VHOST_MSG_RESULT_ERR;
 231
 232         return RTE_VHOST_MSG_RESULT_OK;
 233 }
 234
 235 static int
 236 vhost_user_reset_owner(struct virtio_net **pdev,
 237                         struct VhostUserMsg *msg,
 238                         int main_fd __rte_unused)
 239 {
 240         struct virtio_net *dev = *pdev;
 241
 242         if (validate_msg_fds(msg, 0) != 0)
 243                 return RTE_VHOST_MSG_RESULT_ERR;
 244
 245         vhost_destroy_device_notify(dev);
 246
 247         cleanup_device(dev, 0);
 248         reset_device(dev);
 249         return RTE_VHOST_MSG_RESULT_OK;
 250 }
 251
 252 /*
 253  * The features that we support are requested.
 254  */
 255 static int
 256 vhost_user_get_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
 257                         int main_fd __rte_unused)
 258 {
 259         struct virtio_net *dev = *pdev;
 260         uint64_t features = 0;
 261
 262         if (validate_msg_fds(msg, 0) != 0)
 263                 return RTE_VHOST_MSG_RESULT_ERR;
 264
 265         rte_vhost_driver_get_features(dev->ifname, &features);
 266
 267         msg->payload.u64 = features;
 268         msg->size = sizeof(msg->payload.u64);
 269         msg->fd_num = 0;
 270
 271         return RTE_VHOST_MSG_RESULT_REPLY;
 272 }
 273
 274 /*
 275  * The queue number that we support are requested.
 276  */
 277 static int
 278 vhost_user_get_queue_num(struct virtio_net **pdev, struct VhostUserMsg *msg,
 279                         int main_fd __rte_unused)
 280 {
 281         struct virtio_net *dev = *pdev;
 282         uint32_t queue_num = 0;
 283
 284         if (validate_msg_fds(msg, 0) != 0)
 285                 return RTE_VHOST_MSG_RESULT_ERR;
 286
 287         rte_vhost_driver_get_queue_num(dev->ifname, &queue_num);
 288
 289         msg->payload.u64 = (uint64_t)queue_num;
 290         msg->size = sizeof(msg->payload.u64);
 291         msg->fd_num = 0;
 292
 293         return RTE_VHOST_MSG_RESULT_REPLY;
 294 }
 295
 296 /*
 297  * We receive the negotiated features supported by us and the virtio device.
 298  */
 299 static int
 300 vhost_user_set_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
 301                         int main_fd __rte_unused)
 302 {
 303         struct virtio_net *dev = *pdev;
 304         uint64_t features = msg->payload.u64;
 305         uint64_t vhost_features = 0;
 306         struct rte_vdpa_device *vdpa_dev;
 307
 308         if (validate_msg_fds(msg, 0) != 0)
 309                 return RTE_VHOST_MSG_RESULT_ERR;
 310
 311         rte_vhost_driver_get_features(dev->ifname, &vhost_features);
 312         if (features & ~vhost_features) {
 313                 VHOST_LOG_CONFIG(ERR,
 314                         "(%d) received invalid negotiated features.\n",
 315                         dev->vid);
 316                 dev->flags |= VIRTIO_DEV_FEATURES_FAILED;
 317                 dev->status &= ~VIRTIO_DEVICE_STATUS_FEATURES_OK;
 318
 319                 return RTE_VHOST_MSG_RESULT_ERR;
 320         }
 321
 322         if (dev->flags & VIRTIO_DEV_RUNNING) {
 323                 if (dev->features == features)
 324                         return RTE_VHOST_MSG_RESULT_OK;
 325
 326                 /*
 327                  * Error out if master tries to change features while device is
 328                  * in running state. The exception being VHOST_F_LOG_ALL, which
 329                  * is enabled when the live-migration starts.
 330                  */
 331                 if ((dev->features ^ features) & ~(1ULL << VHOST_F_LOG_ALL)) {
 332                         VHOST_LOG_CONFIG(ERR,
 333                                 "(%d) features changed while device is running.\n",
 334                                 dev->vid);
 335                         return RTE_VHOST_MSG_RESULT_ERR;
 336                 }
 337
 338                 if (dev->notify_ops->features_changed)
 339                         dev->notify_ops->features_changed(dev->vid, features);
 340         }
 341
 342         dev->features = features;
 343         if (dev->features &
 344                 ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
 345                  (1ULL << VIRTIO_F_VERSION_1) |
 346                  (1ULL << VIRTIO_F_RING_PACKED))) {
 347                 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 348         } else {
 349                 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
 350         }
 351         VHOST_LOG_CONFIG(INFO,
 352                 "negotiated Virtio features: 0x%" PRIx64 "\n", dev->features);
 353         VHOST_LOG_CONFIG(DEBUG,
 354                 "(%d) mergeable RX buffers %s, virtio 1 %s\n",
 355                 dev->vid,
 356                 (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
 357                 (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
 358
 359         if ((dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) &&
 360             !(dev->features & (1ULL << VIRTIO_NET_F_MQ))) {
 361                 /*
 362                  * Remove all but first queue pair if MQ hasn't been
 363                  * negotiated. This is safe because the device is not
 364                  * running at this stage.
 365                  */
 366                 while (dev->nr_vring > 2) {
 367                         struct vhost_virtqueue *vq;
 368
 369                         vq = dev->virtqueue[--dev->nr_vring];
 370                         if (!vq)
 371                                 continue;
 372
 373                         dev->virtqueue[dev->nr_vring] = NULL;
 374                         cleanup_vq(vq, 1);
 375                         cleanup_vq_inflight(dev, vq);
 376                         free_vq(dev, vq);
 377                 }
 378         }
 379
 380         vdpa_dev = dev->vdpa_dev;
 381         if (vdpa_dev)
 382                 vdpa_dev->ops->set_features(dev->vid);
 383
 384         dev->flags &= ~VIRTIO_DEV_FEATURES_FAILED;
 385         return RTE_VHOST_MSG_RESULT_OK;
 386 }
 387
 388 /*
 389  * The virtio device sends us the size of the descriptor ring.
 390  */
 391 static int
 392 vhost_user_set_vring_num(struct virtio_net **pdev,
 393                         struct VhostUserMsg *msg,
 394                         int main_fd __rte_unused)
 395 {
 396         struct virtio_net *dev = *pdev;
 397         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
 398
 399         if (validate_msg_fds(msg, 0) != 0)
 400                 return RTE_VHOST_MSG_RESULT_ERR;
 401
 402         vq->size = msg->payload.state.num;
 403
 404         /* VIRTIO 1.0, 2.4 Virtqueues says:
 405          *
 406          *   Queue Size value is always a power of 2. The maximum Queue Size
 407          *   value is 32768.
 408          *
 409          * VIRTIO 1.1 2.7 Virtqueues says:
 410          *
 411          *   Packed virtqueues support up to 2^15 entries each.
 412          */
 413         if (!vq_is_packed(dev)) {
 414                 if (vq->size & (vq->size - 1)) {
 415                         VHOST_LOG_CONFIG(ERR,
 416                                 "invalid virtqueue size %u\n", vq->size);
 417                         return RTE_VHOST_MSG_RESULT_ERR;
 418                 }
 419         }
 420
 421         if (vq->size > 32768) {
 422                 VHOST_LOG_CONFIG(ERR,
 423                         "invalid virtqueue size %u\n", vq->size);
 424                 return RTE_VHOST_MSG_RESULT_ERR;
 425         }
 426
 427         if (vq_is_packed(dev)) {
 428                 if (vq->shadow_used_packed)
 429                         rte_free(vq->shadow_used_packed);
 430                 vq->shadow_used_packed = rte_malloc(NULL,
 431                                 vq->size *
 432                                 sizeof(struct vring_used_elem_packed),
 433                                 RTE_CACHE_LINE_SIZE);
 434                 if (!vq->shadow_used_packed) {
 435                         VHOST_LOG_CONFIG(ERR,
 436                                         "failed to allocate memory for shadow used ring.\n");
 437                         return RTE_VHOST_MSG_RESULT_ERR;
 438                 }
 439
 440         } else {
 441                 if (vq->shadow_used_split)
 442                         rte_free(vq->shadow_used_split);
 443
 444                 vq->shadow_used_split = rte_malloc(NULL,
 445                                 vq->size * sizeof(struct vring_used_elem),
 446                                 RTE_CACHE_LINE_SIZE);
 447
 448                 if (!vq->shadow_used_split) {
 449                         VHOST_LOG_CONFIG(ERR,
 450                                         "failed to allocate memory for vq internal data.\n");
 451                         return RTE_VHOST_MSG_RESULT_ERR;
 452                 }
 453         }
 454
 455         if (vq->batch_copy_elems)
 456                 rte_free(vq->batch_copy_elems);
 457         vq->batch_copy_elems = rte_malloc(NULL,
 458                                 vq->size * sizeof(struct batch_copy_elem),
 459                                 RTE_CACHE_LINE_SIZE);
 460         if (!vq->batch_copy_elems) {
 461                 VHOST_LOG_CONFIG(ERR,
 462                         "failed to allocate memory for batching copy.\n");
 463                 return RTE_VHOST_MSG_RESULT_ERR;
 464         }
 465
 466         return RTE_VHOST_MSG_RESULT_OK;
 467 }
 468
 469 /*
 470  * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the
 471  * same numa node as the memory of vring descriptor.
 472  */
 473 #ifdef RTE_LIBRTE_VHOST_NUMA
 474 static struct virtio_net*
 475 numa_realloc(struct virtio_net *dev, int index)
 476 {
 477         int oldnode, newnode;
 478         struct virtio_net *old_dev;
 479         struct vhost_virtqueue *old_vq, *vq;
 480         struct vring_used_elem *new_shadow_used_split;
 481         struct vring_used_elem_packed *new_shadow_used_packed;
 482         struct batch_copy_elem *new_batch_copy_elems;
 483         int ret;
 484
 485         if (dev->flags & VIRTIO_DEV_RUNNING)
 486                 return dev;
 487
 488         old_dev = dev;
 489         vq = old_vq = dev->virtqueue[index];
 490
 491         ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc,
 492                             MPOL_F_NODE | MPOL_F_ADDR);
 493
 494         /* check if we need to reallocate vq */
 495         ret |= get_mempolicy(&oldnode, NULL, 0, old_vq,
 496                              MPOL_F_NODE | MPOL_F_ADDR);
 497         if (ret) {
 498                 VHOST_LOG_CONFIG(ERR,
 499                         "Unable to get vq numa information.\n");
 500                 return dev;
 501         }
 502         if (oldnode != newnode) {
 503                 VHOST_LOG_CONFIG(INFO,
 504                         "reallocate vq from %d to %d node\n", oldnode, newnode);
 505                 vq = rte_malloc_socket(NULL, sizeof(*vq), 0, newnode);
 506                 if (!vq)
 507                         return dev;
 508
 509                 memcpy(vq, old_vq, sizeof(*vq));
 510
 511                 if (vq_is_packed(dev)) {
 512                         new_shadow_used_packed = rte_malloc_socket(NULL,
 513                                         vq->size *
 514                                         sizeof(struct vring_used_elem_packed),
 515                                         RTE_CACHE_LINE_SIZE,
 516                                         newnode);
 517                         if (new_shadow_used_packed) {
 518                                 rte_free(vq->shadow_used_packed);
 519                                 vq->shadow_used_packed = new_shadow_used_packed;
 520                         }
 521                 } else {
 522                         new_shadow_used_split = rte_malloc_socket(NULL,
 523                                         vq->size *
 524                                         sizeof(struct vring_used_elem),
 525                                         RTE_CACHE_LINE_SIZE,
 526                                         newnode);
 527                         if (new_shadow_used_split) {
 528                                 rte_free(vq->shadow_used_split);
 529                                 vq->shadow_used_split = new_shadow_used_split;
 530                         }
 531                 }
 532
 533                 new_batch_copy_elems = rte_malloc_socket(NULL,
 534                         vq->size * sizeof(struct batch_copy_elem),
 535                         RTE_CACHE_LINE_SIZE,
 536                         newnode);
 537                 if (new_batch_copy_elems) {
 538                         rte_free(vq->batch_copy_elems);
 539                         vq->batch_copy_elems = new_batch_copy_elems;
 540                 }
 541
 542                 rte_free(old_vq);
 543         }
 544
 545         /* check if we need to reallocate dev */
 546         ret = get_mempolicy(&oldnode, NULL, 0, old_dev,
 547                             MPOL_F_NODE | MPOL_F_ADDR);
 548         if (ret) {
 549                 VHOST_LOG_CONFIG(ERR,
 550                         "Unable to get dev numa information.\n");
 551                 goto out;
 552         }
 553         if (oldnode != newnode) {
 554                 VHOST_LOG_CONFIG(INFO,
 555                         "reallocate dev from %d to %d node\n",
 556                         oldnode, newnode);
 557                 dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode);
 558                 if (!dev) {
 559                         dev = old_dev;
 560                         goto out;
 561                 }
 562
 563                 memcpy(dev, old_dev, sizeof(*dev));
 564                 rte_free(old_dev);
 565         }
 566
 567 out:
 568         dev->virtqueue[index] = vq;
 569         vhost_devices[dev->vid] = dev;
 570
 571         if (old_vq != vq)
 572                 vhost_user_iotlb_init(dev, index);
 573
 574         return dev;
 575 }
 576 #else
 577 static struct virtio_net*
 578 numa_realloc(struct virtio_net *dev, int index __rte_unused)
 579 {
 580         return dev;
 581 }
 582 #endif
 583
 584 /* Converts QEMU virtual address to Vhost virtual address. */
 585 static uint64_t
 586 qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
 587 {
 588         struct rte_vhost_mem_region *r;
 589         uint32_t i;
 590
 591         if (unlikely(!dev || !dev->mem))
 592                 goto out_error;
 593
 594         /* Find the region where the address lives. */
 595         for (i = 0; i < dev->mem->nregions; i++) {
 596                 r = &dev->mem->regions[i];
 597
 598                 if (qva >= r->guest_user_addr &&
 599                     qva <  r->guest_user_addr + r->size) {
 600
 601                         if (unlikely(*len > r->guest_user_addr + r->size - qva))
 602                                 *len = r->guest_user_addr + r->size - qva;
 603
 604                         return qva - r->guest_user_addr +
 605                                r->host_user_addr;
 606                 }
 607         }
 608 out_error:
 609         *len = 0;
 610
 611         return 0;
 612 }
 613
 614
 615 /*
 616  * Converts ring address to Vhost virtual address.
 617  * If IOMMU is enabled, the ring address is a guest IO virtual address,
 618  * else it is a QEMU virtual address.
 619  */
 620 static uint64_t
 621 ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 622                 uint64_t ra, uint64_t *size)
 623 {
 624         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 625                 uint64_t vva;
 626
 627                 vhost_user_iotlb_rd_lock(vq);
 628                 vva = vhost_iova_to_vva(dev, vq, ra,
 629                                         size, VHOST_ACCESS_RW);
 630                 vhost_user_iotlb_rd_unlock(vq);
 631
 632                 return vva;
 633         }
 634
 635         return qva_to_vva(dev, ra, size);
 636 }
 637
 638 static uint64_t
 639 log_addr_to_gpa(struct virtio_net *dev, struct vhost_virtqueue *vq)
 640 {
 641         uint64_t log_gpa;
 642
 643         vhost_user_iotlb_rd_lock(vq);
 644         log_gpa = translate_log_addr(dev, vq, vq->ring_addrs.log_guest_addr);
 645         vhost_user_iotlb_rd_unlock(vq);
 646
 647         return log_gpa;
 648 }
 649
 650 static struct virtio_net *
 651 translate_ring_addresses(struct virtio_net *dev, int vq_index)
 652 {
 653         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 654         struct vhost_vring_addr *addr = &vq->ring_addrs;
 655         uint64_t len, expected_len;
 656
 657         if (addr->flags & (1 << VHOST_VRING_F_LOG)) {
 658                 vq->log_guest_addr =
 659                         log_addr_to_gpa(dev, vq);
 660                 if (vq->log_guest_addr == 0) {
 661                         VHOST_LOG_CONFIG(DEBUG,
 662                                 "(%d) failed to map log_guest_addr.\n",
 663                                 dev->vid);
 664                         return dev;
 665                 }
 666         }
 667
 668         if (vq_is_packed(dev)) {
 669                 len = sizeof(struct vring_packed_desc) * vq->size;
 670                 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 671                         ring_addr_to_vva(dev, vq, addr->desc_user_addr, &len);
 672                 if (vq->desc_packed == NULL ||
 673                                 len != sizeof(struct vring_packed_desc) *
 674                                 vq->size) {
 675                         VHOST_LOG_CONFIG(DEBUG,
 676                                 "(%d) failed to map desc_packed ring.\n",
 677                                 dev->vid);
 678                         return dev;
 679                 }
 680
 681                 dev = numa_realloc(dev, vq_index);
 682                 vq = dev->virtqueue[vq_index];
 683                 addr = &vq->ring_addrs;
 684
 685                 len = sizeof(struct vring_packed_desc_event);
 686                 vq->driver_event = (struct vring_packed_desc_event *)
 687                                         (uintptr_t)ring_addr_to_vva(dev,
 688                                         vq, addr->avail_user_addr, &len);
 689                 if (vq->driver_event == NULL ||
 690                                 len != sizeof(struct vring_packed_desc_event)) {
 691                         VHOST_LOG_CONFIG(DEBUG,
 692                                 "(%d) failed to find driver area address.\n",
 693                                 dev->vid);
 694                         return dev;
 695                 }
 696
 697                 len = sizeof(struct vring_packed_desc_event);
 698                 vq->device_event = (struct vring_packed_desc_event *)
 699                                         (uintptr_t)ring_addr_to_vva(dev,
 700                                         vq, addr->used_user_addr, &len);
 701                 if (vq->device_event == NULL ||
 702                                 len != sizeof(struct vring_packed_desc_event)) {
 703                         VHOST_LOG_CONFIG(DEBUG,
 704                                 "(%d) failed to find device area address.\n",
 705                                 dev->vid);
 706                         return dev;
 707                 }
 708
 709                 vq->access_ok = 1;
 710                 return dev;
 711         }
 712
 713         /* The addresses are converted from QEMU virtual to Vhost virtual. */
 714         if (vq->desc && vq->avail && vq->used)
 715                 return dev;
 716
 717         len = sizeof(struct vring_desc) * vq->size;
 718         vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev,
 719                         vq, addr->desc_user_addr, &len);
 720         if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) {
 721                 VHOST_LOG_CONFIG(DEBUG,
 722                         "(%d) failed to map desc ring.\n",
 723                         dev->vid);
 724                 return dev;
 725         }
 726
 727         dev = numa_realloc(dev, vq_index);
 728         vq = dev->virtqueue[vq_index];
 729         addr = &vq->ring_addrs;
 730
 731         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
 732         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 733                 len += sizeof(uint16_t);
 734         expected_len = len;
 735         vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev,
 736                         vq, addr->avail_user_addr, &len);
 737         if (vq->avail == 0 || len != expected_len) {
 738                 VHOST_LOG_CONFIG(DEBUG,
 739                         "(%d) failed to map avail ring.\n",
 740                         dev->vid);
 741                 return dev;
 742         }
 743
 744         len = sizeof(struct vring_used) +
 745                 sizeof(struct vring_used_elem) * vq->size;
 746         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 747                 len += sizeof(uint16_t);
 748         expected_len = len;
 749         vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev,
 750                         vq, addr->used_user_addr, &len);
 751         if (vq->used == 0 || len != expected_len) {
 752                 VHOST_LOG_CONFIG(DEBUG,
 753                         "(%d) failed to map used ring.\n",
 754                         dev->vid);
 755                 return dev;
 756         }
 757
 758         if (vq->last_used_idx != vq->used->idx) {
 759                 VHOST_LOG_CONFIG(WARNING,
 760                         "last_used_idx (%u) and vq->used->idx (%u) mismatches; "
 761                         "some packets maybe resent for Tx and dropped for Rx\n",
 762                         vq->last_used_idx, vq->used->idx);
 763                 vq->last_used_idx  = vq->used->idx;
 764                 vq->last_avail_idx = vq->used->idx;
 765         }
 766
 767         vq->access_ok = 1;
 768
 769         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address desc: %p\n",
 770                         dev->vid, vq->desc);
 771         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address avail: %p\n",
 772                         dev->vid, vq->avail);
 773         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address used: %p\n",
 774                         dev->vid, vq->used);
 775         VHOST_LOG_CONFIG(DEBUG, "(%d) log_guest_addr: %" PRIx64 "\n",
 776                         dev->vid, vq->log_guest_addr);
 777
 778         return dev;
 779 }
 780
 781 /*
 782  * The virtio device sends us the desc, used and avail ring addresses.
 783  * This function then converts these to our address space.
 784  */
 785 static int
 786 vhost_user_set_vring_addr(struct virtio_net **pdev, struct VhostUserMsg *msg,
 787                         int main_fd __rte_unused)
 788 {
 789         struct virtio_net *dev = *pdev;
 790         struct vhost_virtqueue *vq;
 791         struct vhost_vring_addr *addr = &msg->payload.addr;
 792         bool access_ok;
 793
 794         if (validate_msg_fds(msg, 0) != 0)
 795                 return RTE_VHOST_MSG_RESULT_ERR;
 796
 797         if (dev->mem == NULL)
 798                 return RTE_VHOST_MSG_RESULT_ERR;
 799
 800         /* addr->index refers to the queue index. The txq 1, rxq is 0. */
 801         vq = dev->virtqueue[msg->payload.addr.index];
 802
 803         access_ok = vq->access_ok;
 804
 805         /*
 806          * Rings addresses should not be interpreted as long as the ring is not
 807          * started and enabled
 808          */
 809         memcpy(&vq->ring_addrs, addr, sizeof(*addr));
 810
 811         vring_invalidate(dev, vq);
 812
 813         if ((vq->enabled && (dev->features &
 814                                 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) ||
 815                         access_ok) {
 816                 dev = translate_ring_addresses(dev, msg->payload.addr.index);
 817                 if (!dev)
 818                         return RTE_VHOST_MSG_RESULT_ERR;
 819
 820                 *pdev = dev;
 821         }
 822
 823         return RTE_VHOST_MSG_RESULT_OK;
 824 }
 825
 826 /*
 827  * The virtio device sends us the available ring last used index.
 828  */
 829 static int
 830 vhost_user_set_vring_base(struct virtio_net **pdev,
 831                         struct VhostUserMsg *msg,
 832                         int main_fd __rte_unused)
 833 {
 834         struct virtio_net *dev = *pdev;
 835         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
 836         uint64_t val = msg->payload.state.num;
 837
 838         if (validate_msg_fds(msg, 0) != 0)
 839                 return RTE_VHOST_MSG_RESULT_ERR;
 840
 841         if (vq_is_packed(dev)) {
 842                 /*
 843                  * Bit[0:14]: avail index
 844                  * Bit[15]: avail wrap counter
 845                  */
 846                 vq->last_avail_idx = val & 0x7fff;
 847                 vq->avail_wrap_counter = !!(val & (0x1 << 15));
 848                 /*
 849                  * Set used index to same value as available one, as
 850                  * their values should be the same since ring processing
 851                  * was stopped at get time.
 852                  */
 853                 vq->last_used_idx = vq->last_avail_idx;
 854                 vq->used_wrap_counter = vq->avail_wrap_counter;
 855         } else {
 856                 vq->last_used_idx = msg->payload.state.num;
 857                 vq->last_avail_idx = msg->payload.state.num;
 858         }
 859
 860         return RTE_VHOST_MSG_RESULT_OK;
 861 }
 862
 863 static int
 864 add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
 865                    uint64_t host_phys_addr, uint64_t size)
 866 {
 867         struct guest_page *page, *last_page;
 868         struct guest_page *old_pages;
 869
 870         if (dev->nr_guest_pages == dev->max_guest_pages) {
 871                 dev->max_guest_pages *= 2;
 872                 old_pages = dev->guest_pages;
 873                 dev->guest_pages = rte_realloc(dev->guest_pages,
 874                                         dev->max_guest_pages * sizeof(*page),
 875                                         RTE_CACHE_LINE_SIZE);
 876                 if (dev->guest_pages == NULL) {
 877                         VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
 878                         rte_free(old_pages);
 879                         return -1;
 880                 }
 881         }
 882
 883         if (dev->nr_guest_pages > 0) {
 884                 last_page = &dev->guest_pages[dev->nr_guest_pages - 1];
 885                 /* merge if the two pages are continuous */
 886                 if (host_phys_addr == last_page->host_phys_addr +
 887                                       last_page->size) {
 888                         last_page->size += size;
 889                         return 0;
 890                 }
 891         }
 892
 893         page = &dev->guest_pages[dev->nr_guest_pages++];
 894         page->guest_phys_addr = guest_phys_addr;
 895         page->host_phys_addr  = host_phys_addr;
 896         page->size = size;
 897
 898         return 0;
 899 }
 900
 901 static int
 902 add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
 903                 uint64_t page_size)
 904 {
 905         uint64_t reg_size = reg->size;
 906         uint64_t host_user_addr  = reg->host_user_addr;
 907         uint64_t guest_phys_addr = reg->guest_phys_addr;
 908         uint64_t host_phys_addr;
 909         uint64_t size;
 910
 911         host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr);
 912         size = page_size - (guest_phys_addr & (page_size - 1));
 913         size = RTE_MIN(size, reg_size);
 914
 915         if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size) < 0)
 916                 return -1;
 917
 918         host_user_addr  += size;
 919         guest_phys_addr += size;
 920         reg_size -= size;
 921
 922         while (reg_size > 0) {
 923                 size = RTE_MIN(reg_size, page_size);
 924                 host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)
 925                                                   host_user_addr);
 926                 if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr,
 927                                 size) < 0)
 928                         return -1;
 929
 930                 host_user_addr  += size;
 931                 guest_phys_addr += size;
 932                 reg_size -= size;
 933         }
 934
 935         /* sort guest page array if over binary search threshold */
 936         if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
 937                 qsort((void *)dev->guest_pages, dev->nr_guest_pages,
 938                         sizeof(struct guest_page), guest_page_addrcmp);
 939         }
 940
 941         return 0;
 942 }
 943
 944 #ifdef RTE_LIBRTE_VHOST_DEBUG
 945 /* TODO: enable it only in debug mode? */
 946 static void
 947 dump_guest_pages(struct virtio_net *dev)
 948 {
 949         uint32_t i;
 950         struct guest_page *page;
 951
 952         for (i = 0; i < dev->nr_guest_pages; i++) {
 953                 page = &dev->guest_pages[i];
 954
 955                 VHOST_LOG_CONFIG(INFO,
 956                         "guest physical page region %u\n"
 957                         "\t guest_phys_addr: %" PRIx64 "\n"
 958                         "\t host_phys_addr : %" PRIx64 "\n"
 959                         "\t size           : %" PRIx64 "\n",
 960                         i,
 961                         page->guest_phys_addr,
 962                         page->host_phys_addr,
 963                         page->size);
 964         }
 965 }
 966 #else
 967 #define dump_guest_pages(dev)
 968 #endif
 969
 970 static bool
 971 vhost_memory_changed(struct VhostUserMemory *new,
 972                      struct rte_vhost_memory *old)
 973 {
 974         uint32_t i;
 975
 976         if (new->nregions != old->nregions)
 977                 return true;
 978
 979         for (i = 0; i < new->nregions; ++i) {
 980                 VhostUserMemoryRegion *new_r = &new->regions[i];
 981                 struct rte_vhost_mem_region *old_r = &old->regions[i];
 982
 983                 if (new_r->guest_phys_addr != old_r->guest_phys_addr)
 984                         return true;
 985                 if (new_r->memory_size != old_r->size)
 986                         return true;
 987                 if (new_r->userspace_addr != old_r->guest_user_addr)
 988                         return true;
 989         }
 990
 991         return false;
 992 }
 993
 994 static int
 995 vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *msg,
 996                         int main_fd)
 997 {
 998         struct virtio_net *dev = *pdev;
 999         struct VhostUserMemory *memory = &msg->payload.memory;
1000         struct rte_vhost_mem_region *reg;
1001         void *mmap_addr;
1002         uint64_t mmap_size;
1003         uint64_t mmap_offset;
1004         uint64_t alignment;
1005         uint32_t i;
1006         int populate;
1007         int fd;
1008
1009         if (validate_msg_fds(msg, memory->nregions) != 0)
1010                 return RTE_VHOST_MSG_RESULT_ERR;
1011
1012         if (memory->nregions > VHOST_MEMORY_MAX_NREGIONS) {
1013                 VHOST_LOG_CONFIG(ERR,
1014                         "too many memory regions (%u)\n", memory->nregions);
1015                 return RTE_VHOST_MSG_RESULT_ERR;
1016         }
1017
1018         if (dev->mem && !vhost_memory_changed(memory, dev->mem)) {
1019                 VHOST_LOG_CONFIG(INFO,
1020                         "(%d) memory regions not changed\n", dev->vid);
1021
1022                 close_msg_fds(msg);
1023
1024                 return RTE_VHOST_MSG_RESULT_OK;
1025         }
1026
1027         if (dev->mem) {
1028                 if (dev->flags & VIRTIO_DEV_VDPA_CONFIGURED) {
1029                         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
1030
1031                         if (vdpa_dev && vdpa_dev->ops->dev_close)
1032                                 vdpa_dev->ops->dev_close(dev->vid);
1033                         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
1034                 }
1035                 free_mem_region(dev);
1036                 rte_free(dev->mem);
1037                 dev->mem = NULL;
1038         }
1039
1040         /* Flush IOTLB cache as previous HVAs are now invalid */
1041         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1042                 for (i = 0; i < dev->nr_vring; i++)
1043                         vhost_user_iotlb_flush_all(dev->virtqueue[i]);
1044
1045         dev->nr_guest_pages = 0;
1046         if (dev->guest_pages == NULL) {
1047                 dev->max_guest_pages = 8;
1048                 dev->guest_pages = rte_zmalloc(NULL,
1049                                         dev->max_guest_pages *
1050                                         sizeof(struct guest_page),
1051                                         RTE_CACHE_LINE_SIZE);
1052                 if (dev->guest_pages == NULL) {
1053                         VHOST_LOG_CONFIG(ERR,
1054                                 "(%d) failed to allocate memory "
1055                                 "for dev->guest_pages\n",
1056                                 dev->vid);
1057                         return RTE_VHOST_MSG_RESULT_ERR;
1058                 }
1059         }
1060
1061         dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct rte_vhost_memory) +
1062                 sizeof(struct rte_vhost_mem_region) * memory->nregions, 0);
1063         if (dev->mem == NULL) {
1064                 VHOST_LOG_CONFIG(ERR,
1065                         "(%d) failed to allocate memory for dev->mem\n",
1066                         dev->vid);
1067                 return RTE_VHOST_MSG_RESULT_ERR;
1068         }
1069         dev->mem->nregions = memory->nregions;
1070
1071         for (i = 0; i < memory->nregions; i++) {
1072                 fd  = msg->fds[i];
1073                 reg = &dev->mem->regions[i];
1074
1075                 reg->guest_phys_addr = memory->regions[i].guest_phys_addr;
1076                 reg->guest_user_addr = memory->regions[i].userspace_addr;
1077                 reg->size            = memory->regions[i].memory_size;
1078                 reg->fd              = fd;
1079
1080                 mmap_offset = memory->regions[i].mmap_offset;
1081
1082                 /* Check for memory_size + mmap_offset overflow */
1083                 if (mmap_offset >= -reg->size) {
1084                         VHOST_LOG_CONFIG(ERR,
1085                                 "mmap_offset (%#"PRIx64") and memory_size "
1086                                 "(%#"PRIx64") overflow\n",
1087                                 mmap_offset, reg->size);
1088                         goto err_mmap;
1089                 }
1090
1091                 mmap_size = reg->size + mmap_offset;
1092
1093                 /* mmap() without flag of MAP_ANONYMOUS, should be called
1094                  * with length argument aligned with hugepagesz at older
1095                  * longterm version Linux, like 2.6.32 and 3.2.72, or
1096                  * mmap() will fail with EINVAL.
1097                  *
1098                  * to avoid failure, make sure in caller to keep length
1099                  * aligned.
1100                  */
1101                 alignment = get_blk_size(fd);
1102                 if (alignment == (uint64_t)-1) {
1103                         VHOST_LOG_CONFIG(ERR,
1104                                 "couldn't get hugepage size through fstat\n");
1105                         goto err_mmap;
1106                 }
1107                 mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
1108                 if (mmap_size == 0) {
1109                         /*
1110                          * It could happen if initial mmap_size + alignment
1111                          * overflows the sizeof uint64, which could happen if
1112                          * either mmap_size or alignment value is wrong.
1113                          *
1114                          * mmap() kernel implementation would return an error,
1115                          * but better catch it before and provide useful info
1116                          * in the logs.
1117                          */
1118                         VHOST_LOG_CONFIG(ERR, "mmap size (0x%" PRIx64 ") "
1119                                         "or alignment (0x%" PRIx64 ") is invalid\n",
1120                                         reg->size + mmap_offset, alignment);
1121                         goto err_mmap;
1122                 }
1123
1124                 populate = dev->async_copy ? MAP_POPULATE : 0;
1125                 mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
1126                                  MAP_SHARED | populate, fd, 0);
1127
1128                 if (mmap_addr == MAP_FAILED) {
1129                         VHOST_LOG_CONFIG(ERR,
1130                                 "mmap region %u failed.\n", i);
1131                         goto err_mmap;
1132                 }
1133
1134                 reg->mmap_addr = mmap_addr;
1135                 reg->mmap_size = mmap_size;
1136                 reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr +
1137                                       mmap_offset;
1138
1139                 if (dev->async_copy)
1140                         if (add_guest_pages(dev, reg, alignment) < 0) {
1141                                 VHOST_LOG_CONFIG(ERR,
1142                                         "adding guest pages to region %u failed.\n",
1143                                         i);
1144                                 goto err_mmap;
1145                         }
1146
1147                 VHOST_LOG_CONFIG(INFO,
1148                         "guest memory region %u, size: 0x%" PRIx64 "\n"
1149                         "\t guest physical addr: 0x%" PRIx64 "\n"
1150                         "\t guest virtual  addr: 0x%" PRIx64 "\n"
1151                         "\t host  virtual  addr: 0x%" PRIx64 "\n"
1152                         "\t mmap addr : 0x%" PRIx64 "\n"
1153                         "\t mmap size : 0x%" PRIx64 "\n"
1154                         "\t mmap align: 0x%" PRIx64 "\n"
1155                         "\t mmap off  : 0x%" PRIx64 "\n",
1156                         i, reg->size,
1157                         reg->guest_phys_addr,
1158                         reg->guest_user_addr,
1159                         reg->host_user_addr,
1160                         (uint64_t)(uintptr_t)mmap_addr,
1161                         mmap_size,
1162                         alignment,
1163                         mmap_offset);
1164
1165                 if (dev->postcopy_listening) {
1166                         /*
1167                          * We haven't a better way right now than sharing
1168                          * DPDK's virtual address with Qemu, so that Qemu can
1169                          * retrieve the region offset when handling userfaults.
1170                          */
1171                         memory->regions[i].userspace_addr =
1172                                 reg->host_user_addr;
1173                 }
1174         }
1175         if (dev->postcopy_listening) {
1176                 /* Send the addresses back to qemu */
1177                 msg->fd_num = 0;
1178                 send_vhost_reply(main_fd, msg);
1179
1180                 /* Wait for qemu to acknolwedge it's got the addresses
1181                  * we've got to wait before we're allowed to generate faults.
1182                  */
1183                 VhostUserMsg ack_msg;
1184                 if (read_vhost_message(main_fd, &ack_msg) <= 0) {
1185                         VHOST_LOG_CONFIG(ERR,
1186                                 "Failed to read qemu ack on postcopy set-mem-table\n");
1187                         goto err_mmap;
1188                 }
1189
1190                 if (validate_msg_fds(&ack_msg, 0) != 0)
1191                         goto err_mmap;
1192
1193                 if (ack_msg.request.master != VHOST_USER_SET_MEM_TABLE) {
1194                         VHOST_LOG_CONFIG(ERR,
1195                                 "Bad qemu ack on postcopy set-mem-table (%d)\n",
1196                                 ack_msg.request.master);
1197                         goto err_mmap;
1198                 }
1199
1200                 /* Now userfault register and we can use the memory */
1201                 for (i = 0; i < memory->nregions; i++) {
1202 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
1203                         reg = &dev->mem->regions[i];
1204                         struct uffdio_register reg_struct;
1205
1206                         /*
1207                          * Let's register all the mmap'ed area to ensure
1208                          * alignment on page boundary.
1209                          */
1210                         reg_struct.range.start =
1211                                 (uint64_t)(uintptr_t)reg->mmap_addr;
1212                         reg_struct.range.len = reg->mmap_size;
1213                         reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
1214
1215                         if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER,
1216                                                 &reg_struct)) {
1217                                 VHOST_LOG_CONFIG(ERR,
1218                                         "Failed to register ufd for region %d: (ufd = %d) %s\n",
1219                                         i, dev->postcopy_ufd,
1220                                         strerror(errno));
1221                                 goto err_mmap;
1222                         }
1223                         VHOST_LOG_CONFIG(INFO,
1224                                 "\t userfaultfd registered for range : "
1225                                 "%" PRIx64 " - %" PRIx64 "\n",
1226                                 (uint64_t)reg_struct.range.start,
1227                                 (uint64_t)reg_struct.range.start +
1228                                 (uint64_t)reg_struct.range.len - 1);
1229 #else
1230                         goto err_mmap;
1231 #endif
1232                 }
1233         }
1234
1235         for (i = 0; i < dev->nr_vring; i++) {
1236                 struct vhost_virtqueue *vq = dev->virtqueue[i];
1237
1238                 if (!vq)
1239                         continue;
1240
1241                 if (vq->desc || vq->avail || vq->used) {
1242                         /*
1243                          * If the memory table got updated, the ring addresses
1244                          * need to be translated again as virtual addresses have
1245                          * changed.
1246                          */
1247                         vring_invalidate(dev, vq);
1248
1249                         dev = translate_ring_addresses(dev, i);
1250                         if (!dev) {
1251                                 dev = *pdev;
1252                                 goto err_mmap;
1253                         }
1254
1255                         *pdev = dev;
1256                 }
1257         }
1258
1259         dump_guest_pages(dev);
1260
1261         return RTE_VHOST_MSG_RESULT_OK;
1262
1263 err_mmap:
1264         free_mem_region(dev);
1265         rte_free(dev->mem);
1266         dev->mem = NULL;
1267         return RTE_VHOST_MSG_RESULT_ERR;
1268 }
1269
1270 static bool
1271 vq_is_ready(struct virtio_net *dev, struct vhost_virtqueue *vq)
1272 {
1273         bool rings_ok;
1274
1275         if (!vq)
1276                 return false;
1277
1278         if (vq_is_packed(dev))
1279                 rings_ok = vq->desc_packed && vq->driver_event &&
1280                         vq->device_event;
1281         else
1282                 rings_ok = vq->desc && vq->avail && vq->used;
1283
1284         return rings_ok &&
1285                vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1286                vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1287                vq->enabled;
1288 }
1289
1290 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
1291
1292 static int
1293 virtio_is_ready(struct virtio_net *dev)
1294 {
1295         struct vhost_virtqueue *vq;
1296         uint32_t i, nr_vring = dev->nr_vring;
1297
1298         if (dev->flags & VIRTIO_DEV_READY)
1299                 return 1;
1300
1301         if (!dev->nr_vring)
1302                 return 0;
1303
1304         if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
1305                 nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
1306
1307                 if (dev->nr_vring < nr_vring)
1308                         return 0;
1309         }
1310
1311         for (i = 0; i < nr_vring; i++) {
1312                 vq = dev->virtqueue[i];
1313
1314                 if (!vq_is_ready(dev, vq))
1315                         return 0;
1316         }
1317
1318         /* If supported, ensure the frontend is really done with config */
1319         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
1320                 if (!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK))
1321                         return 0;
1322
1323         dev->flags |= VIRTIO_DEV_READY;
1324
1325         if (!(dev->flags & VIRTIO_DEV_RUNNING))
1326                 VHOST_LOG_CONFIG(INFO,
1327                         "virtio is now ready for processing.\n");
1328         return 1;
1329 }
1330
1331 static void *
1332 inflight_mem_alloc(const char *name, size_t size, int *fd)
1333 {
1334         void *ptr;
1335         int mfd = -1;
1336         char fname[20] = "/tmp/memfd-XXXXXX";
1337
1338         *fd = -1;
1339 #ifdef MEMFD_SUPPORTED
1340         mfd = memfd_create(name, MFD_CLOEXEC);
1341 #else
1342         RTE_SET_USED(name);
1343 #endif
1344         if (mfd == -1) {
1345                 mfd = mkstemp(fname);
1346                 if (mfd == -1) {
1347                         VHOST_LOG_CONFIG(ERR,
1348                                 "failed to get inflight buffer fd\n");
1349                         return NULL;
1350                 }
1351
1352                 unlink(fname);
1353         }
1354
1355         if (ftruncate(mfd, size) == -1) {
1356                 VHOST_LOG_CONFIG(ERR,
1357                         "failed to alloc inflight buffer\n");
1358                 close(mfd);
1359                 return NULL;
1360         }
1361
1362         ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
1363         if (ptr == MAP_FAILED) {
1364                 VHOST_LOG_CONFIG(ERR,
1365                         "failed to mmap inflight buffer\n");
1366                 close(mfd);
1367                 return NULL;
1368         }
1369
1370         *fd = mfd;
1371         return ptr;
1372 }
1373
1374 static uint32_t
1375 get_pervq_shm_size_split(uint16_t queue_size)
1376 {
1377         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_split) *
1378                                   queue_size + sizeof(uint64_t) +
1379                                   sizeof(uint16_t) * 4, INFLIGHT_ALIGNMENT);
1380 }
1381
1382 static uint32_t
1383 get_pervq_shm_size_packed(uint16_t queue_size)
1384 {
1385         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_packed)
1386                                   * queue_size + sizeof(uint64_t) +
1387                                   sizeof(uint16_t) * 6 + sizeof(uint8_t) * 9,
1388                                   INFLIGHT_ALIGNMENT);
1389 }
1390
1391 static int
1392 vhost_user_get_inflight_fd(struct virtio_net **pdev,
1393                            VhostUserMsg *msg,
1394                            int main_fd __rte_unused)
1395 {
1396         struct rte_vhost_inflight_info_packed *inflight_packed;
1397         uint64_t pervq_inflight_size, mmap_size;
1398         uint16_t num_queues, queue_size;
1399         struct virtio_net *dev = *pdev;
1400         int fd, i, j;
1401         void *addr;
1402
1403         if (msg->size != sizeof(msg->payload.inflight)) {
1404                 VHOST_LOG_CONFIG(ERR,
1405                         "invalid get_inflight_fd message size is %d\n",
1406                         msg->size);
1407                 return RTE_VHOST_MSG_RESULT_ERR;
1408         }
1409
1410         if (dev->inflight_info == NULL) {
1411                 dev->inflight_info = calloc(1,
1412                                             sizeof(struct inflight_mem_info));
1413                 if (!dev->inflight_info) {
1414                         VHOST_LOG_CONFIG(ERR,
1415                                 "failed to alloc dev inflight area\n");
1416                         return RTE_VHOST_MSG_RESULT_ERR;
1417                 }
1418                 dev->inflight_info->fd = -1;
1419         }
1420
1421         num_queues = msg->payload.inflight.num_queues;
1422         queue_size = msg->payload.inflight.queue_size;
1423
1424         VHOST_LOG_CONFIG(INFO, "get_inflight_fd num_queues: %u\n",
1425                 msg->payload.inflight.num_queues);
1426         VHOST_LOG_CONFIG(INFO, "get_inflight_fd queue_size: %u\n",
1427                 msg->payload.inflight.queue_size);
1428
1429         if (vq_is_packed(dev))
1430                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1431         else
1432                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1433
1434         mmap_size = num_queues * pervq_inflight_size;
1435         addr = inflight_mem_alloc("vhost-inflight", mmap_size, &fd);
1436         if (!addr) {
1437                 VHOST_LOG_CONFIG(ERR,
1438                         "failed to alloc vhost inflight area\n");
1439                         msg->payload.inflight.mmap_size = 0;
1440                 return RTE_VHOST_MSG_RESULT_ERR;
1441         }
1442         memset(addr, 0, mmap_size);
1443
1444         if (dev->inflight_info->addr) {
1445                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1446                 dev->inflight_info->addr = NULL;
1447         }
1448
1449         if (dev->inflight_info->fd >= 0) {
1450                 close(dev->inflight_info->fd);
1451                 dev->inflight_info->fd = -1;
1452         }
1453
1454         dev->inflight_info->addr = addr;
1455         dev->inflight_info->size = msg->payload.inflight.mmap_size = mmap_size;
1456         dev->inflight_info->fd = msg->fds[0] = fd;
1457         msg->payload.inflight.mmap_offset = 0;
1458         msg->fd_num = 1;
1459
1460         if (vq_is_packed(dev)) {
1461                 for (i = 0; i < num_queues; i++) {
1462                         inflight_packed =
1463                                 (struct rte_vhost_inflight_info_packed *)addr;
1464                         inflight_packed->used_wrap_counter = 1;
1465                         inflight_packed->old_used_wrap_counter = 1;
1466                         for (j = 0; j < queue_size; j++)
1467                                 inflight_packed->desc[j].next = j + 1;
1468                         addr = (void *)((char *)addr + pervq_inflight_size);
1469                 }
1470         }
1471
1472         VHOST_LOG_CONFIG(INFO,
1473                 "send inflight mmap_size: %"PRIu64"\n",
1474                 msg->payload.inflight.mmap_size);
1475         VHOST_LOG_CONFIG(INFO,
1476                 "send inflight mmap_offset: %"PRIu64"\n",
1477                 msg->payload.inflight.mmap_offset);
1478         VHOST_LOG_CONFIG(INFO,
1479                 "send inflight fd: %d\n", msg->fds[0]);
1480
1481         return RTE_VHOST_MSG_RESULT_REPLY;
1482 }
1483
1484 static int
1485 vhost_user_set_inflight_fd(struct virtio_net **pdev, VhostUserMsg *msg,
1486                            int main_fd __rte_unused)
1487 {
1488         uint64_t mmap_size, mmap_offset;
1489         uint16_t num_queues, queue_size;
1490         struct virtio_net *dev = *pdev;
1491         uint32_t pervq_inflight_size;
1492         struct vhost_virtqueue *vq;
1493         void *addr;
1494         int fd, i;
1495
1496         fd = msg->fds[0];
1497         if (msg->size != sizeof(msg->payload.inflight) || fd < 0) {
1498                 VHOST_LOG_CONFIG(ERR,
1499                         "invalid set_inflight_fd message size is %d,fd is %d\n",
1500                         msg->size, fd);
1501                 return RTE_VHOST_MSG_RESULT_ERR;
1502         }
1503
1504         mmap_size = msg->payload.inflight.mmap_size;
1505         mmap_offset = msg->payload.inflight.mmap_offset;
1506         num_queues = msg->payload.inflight.num_queues;
1507         queue_size = msg->payload.inflight.queue_size;
1508
1509         if (vq_is_packed(dev))
1510                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1511         else
1512                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1513
1514         VHOST_LOG_CONFIG(INFO,
1515                 "set_inflight_fd mmap_size: %"PRIu64"\n", mmap_size);
1516         VHOST_LOG_CONFIG(INFO,
1517                 "set_inflight_fd mmap_offset: %"PRIu64"\n", mmap_offset);
1518         VHOST_LOG_CONFIG(INFO,
1519                 "set_inflight_fd num_queues: %u\n", num_queues);
1520         VHOST_LOG_CONFIG(INFO,
1521                 "set_inflight_fd queue_size: %u\n", queue_size);
1522         VHOST_LOG_CONFIG(INFO,
1523                 "set_inflight_fd fd: %d\n", fd);
1524         VHOST_LOG_CONFIG(INFO,
1525                 "set_inflight_fd pervq_inflight_size: %d\n",
1526                 pervq_inflight_size);
1527
1528         if (!dev->inflight_info) {
1529                 dev->inflight_info = calloc(1,
1530                                             sizeof(struct inflight_mem_info));
1531                 if (dev->inflight_info == NULL) {
1532                         VHOST_LOG_CONFIG(ERR,
1533                                 "failed to alloc dev inflight area\n");
1534                         return RTE_VHOST_MSG_RESULT_ERR;
1535                 }
1536                 dev->inflight_info->fd = -1;
1537         }
1538
1539         if (dev->inflight_info->addr) {
1540                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1541                 dev->inflight_info->addr = NULL;
1542         }
1543
1544         addr = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1545                     fd, mmap_offset);
1546         if (addr == MAP_FAILED) {
1547                 VHOST_LOG_CONFIG(ERR, "failed to mmap share memory.\n");
1548                 return RTE_VHOST_MSG_RESULT_ERR;
1549         }
1550
1551         if (dev->inflight_info->fd >= 0) {
1552                 close(dev->inflight_info->fd);
1553                 dev->inflight_info->fd = -1;
1554         }
1555
1556         dev->inflight_info->fd = fd;
1557         dev->inflight_info->addr = addr;
1558         dev->inflight_info->size = mmap_size;
1559
1560         for (i = 0; i < num_queues; i++) {
1561                 vq = dev->virtqueue[i];
1562                 if (!vq)
1563                         continue;
1564
1565                 if (vq_is_packed(dev)) {
1566                         vq->inflight_packed = addr;
1567                         vq->inflight_packed->desc_num = queue_size;
1568                 } else {
1569                         vq->inflight_split = addr;
1570                         vq->inflight_split->desc_num = queue_size;
1571                 }
1572                 addr = (void *)((char *)addr + pervq_inflight_size);
1573         }
1574
1575         return RTE_VHOST_MSG_RESULT_OK;
1576 }
1577
1578 static int
1579 vhost_user_set_vring_call(struct virtio_net **pdev, struct VhostUserMsg *msg,
1580                         int main_fd __rte_unused)
1581 {
1582         struct virtio_net *dev = *pdev;
1583         struct vhost_vring_file file;
1584         struct vhost_virtqueue *vq;
1585         int expected_fds;
1586
1587         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1588         if (validate_msg_fds(msg, expected_fds) != 0)
1589                 return RTE_VHOST_MSG_RESULT_ERR;
1590
1591         file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1592         if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1593                 file.fd = VIRTIO_INVALID_EVENTFD;
1594         else
1595                 file.fd = msg->fds[0];
1596         VHOST_LOG_CONFIG(INFO,
1597                 "vring call idx:%d file:%d\n", file.index, file.fd);
1598
1599         vq = dev->virtqueue[file.index];
1600
1601         if (vq->ready) {
1602                 vq->ready = 0;
1603                 vhost_user_notify_queue_state(dev, file.index, 0);
1604         }
1605
1606         if (vq->callfd >= 0)
1607                 close(vq->callfd);
1608
1609         vq->callfd = file.fd;
1610
1611         return RTE_VHOST_MSG_RESULT_OK;
1612 }
1613
1614 static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
1615                         struct VhostUserMsg *msg,
1616                         int main_fd __rte_unused)
1617 {
1618         int expected_fds;
1619
1620         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1621         if (validate_msg_fds(msg, expected_fds) != 0)
1622                 return RTE_VHOST_MSG_RESULT_ERR;
1623
1624         if (!(msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1625                 close(msg->fds[0]);
1626         VHOST_LOG_CONFIG(INFO, "not implemented\n");
1627
1628         return RTE_VHOST_MSG_RESULT_OK;
1629 }
1630
1631 static int
1632 resubmit_desc_compare(const void *a, const void *b)
1633 {
1634         const struct rte_vhost_resubmit_desc *desc0 = a;
1635         const struct rte_vhost_resubmit_desc *desc1 = b;
1636
1637         if (desc1->counter > desc0->counter)
1638                 return 1;
1639
1640         return -1;
1641 }
1642
1643 static int
1644 vhost_check_queue_inflights_split(struct virtio_net *dev,
1645                                   struct vhost_virtqueue *vq)
1646 {
1647         uint16_t i;
1648         uint16_t resubmit_num = 0, last_io, num;
1649         struct vring_used *used = vq->used;
1650         struct rte_vhost_resubmit_info *resubmit;
1651         struct rte_vhost_inflight_info_split *inflight_split;
1652
1653         if (!(dev->protocol_features &
1654             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1655                 return RTE_VHOST_MSG_RESULT_OK;
1656
1657         /* The frontend may still not support the inflight feature
1658          * although we negotiate the protocol feature.
1659          */
1660         if ((!vq->inflight_split))
1661                 return RTE_VHOST_MSG_RESULT_OK;
1662
1663         if (!vq->inflight_split->version) {
1664                 vq->inflight_split->version = INFLIGHT_VERSION;
1665                 return RTE_VHOST_MSG_RESULT_OK;
1666         }
1667
1668         if (vq->resubmit_inflight)
1669                 return RTE_VHOST_MSG_RESULT_OK;
1670
1671         inflight_split = vq->inflight_split;
1672         vq->global_counter = 0;
1673         last_io = inflight_split->last_inflight_io;
1674
1675         if (inflight_split->used_idx != used->idx) {
1676                 inflight_split->desc[last_io].inflight = 0;
1677                 rte_smp_mb();
1678                 inflight_split->used_idx = used->idx;
1679         }
1680
1681         for (i = 0; i < inflight_split->desc_num; i++) {
1682                 if (inflight_split->desc[i].inflight == 1)
1683                         resubmit_num++;
1684         }
1685
1686         vq->last_avail_idx += resubmit_num;
1687
1688         if (resubmit_num) {
1689                 resubmit  = calloc(1, sizeof(struct rte_vhost_resubmit_info));
1690                 if (!resubmit) {
1691                         VHOST_LOG_CONFIG(ERR,
1692                                 "failed to allocate memory for resubmit info.\n");
1693                         return RTE_VHOST_MSG_RESULT_ERR;
1694                 }
1695
1696                 resubmit->resubmit_list = calloc(resubmit_num,
1697                         sizeof(struct rte_vhost_resubmit_desc));
1698                 if (!resubmit->resubmit_list) {
1699                         VHOST_LOG_CONFIG(ERR,
1700                                 "failed to allocate memory for inflight desc.\n");
1701                         free(resubmit);
1702                         return RTE_VHOST_MSG_RESULT_ERR;
1703                 }
1704
1705                 num = 0;
1706                 for (i = 0; i < vq->inflight_split->desc_num; i++) {
1707                         if (vq->inflight_split->desc[i].inflight == 1) {
1708                                 resubmit->resubmit_list[num].index = i;
1709                                 resubmit->resubmit_list[num].counter =
1710                                         inflight_split->desc[i].counter;
1711                                 num++;
1712                         }
1713                 }
1714                 resubmit->resubmit_num = num;
1715
1716                 if (resubmit->resubmit_num > 1)
1717                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1718                               sizeof(struct rte_vhost_resubmit_desc),
1719                               resubmit_desc_compare);
1720
1721                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1722                 vq->resubmit_inflight = resubmit;
1723         }
1724
1725         return RTE_VHOST_MSG_RESULT_OK;
1726 }
1727
1728 static int
1729 vhost_check_queue_inflights_packed(struct virtio_net *dev,
1730                                    struct vhost_virtqueue *vq)
1731 {
1732         uint16_t i;
1733         uint16_t resubmit_num = 0, old_used_idx, num;
1734         struct rte_vhost_resubmit_info *resubmit;
1735         struct rte_vhost_inflight_info_packed *inflight_packed;
1736
1737         if (!(dev->protocol_features &
1738             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1739                 return RTE_VHOST_MSG_RESULT_OK;
1740
1741         /* The frontend may still not support the inflight feature
1742          * although we negotiate the protocol feature.
1743          */
1744         if ((!vq->inflight_packed))
1745                 return RTE_VHOST_MSG_RESULT_OK;
1746
1747         if (!vq->inflight_packed->version) {
1748                 vq->inflight_packed->version = INFLIGHT_VERSION;
1749                 return RTE_VHOST_MSG_RESULT_OK;
1750         }
1751
1752         if (vq->resubmit_inflight)
1753                 return RTE_VHOST_MSG_RESULT_OK;
1754
1755         inflight_packed = vq->inflight_packed;
1756         vq->global_counter = 0;
1757         old_used_idx = inflight_packed->old_used_idx;
1758
1759         if (inflight_packed->used_idx != old_used_idx) {
1760                 if (inflight_packed->desc[old_used_idx].inflight == 0) {
1761                         inflight_packed->old_used_idx =
1762                                 inflight_packed->used_idx;
1763                         inflight_packed->old_used_wrap_counter =
1764                                 inflight_packed->used_wrap_counter;
1765                         inflight_packed->old_free_head =
1766                                 inflight_packed->free_head;
1767                 } else {
1768                         inflight_packed->used_idx =
1769                                 inflight_packed->old_used_idx;
1770                         inflight_packed->used_wrap_counter =
1771                                 inflight_packed->old_used_wrap_counter;
1772                         inflight_packed->free_head =
1773                                 inflight_packed->old_free_head;
1774                 }
1775         }
1776
1777         for (i = 0; i < inflight_packed->desc_num; i++) {
1778                 if (inflight_packed->desc[i].inflight == 1)
1779                         resubmit_num++;
1780         }
1781
1782         if (resubmit_num) {
1783                 resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
1784                 if (resubmit == NULL) {
1785                         VHOST_LOG_CONFIG(ERR,
1786                                 "failed to allocate memory for resubmit info.\n");
1787                         return RTE_VHOST_MSG_RESULT_ERR;
1788                 }
1789
1790                 resubmit->resubmit_list = calloc(resubmit_num,
1791                         sizeof(struct rte_vhost_resubmit_desc));
1792                 if (resubmit->resubmit_list == NULL) {
1793                         VHOST_LOG_CONFIG(ERR,
1794                                 "failed to allocate memory for resubmit desc.\n");
1795                         free(resubmit);
1796                         return RTE_VHOST_MSG_RESULT_ERR;
1797                 }
1798
1799                 num = 0;
1800                 for (i = 0; i < inflight_packed->desc_num; i++) {
1801                         if (vq->inflight_packed->desc[i].inflight == 1) {
1802                                 resubmit->resubmit_list[num].index = i;
1803                                 resubmit->resubmit_list[num].counter =
1804                                         inflight_packed->desc[i].counter;
1805                                 num++;
1806                         }
1807                 }
1808                 resubmit->resubmit_num = num;
1809
1810                 if (resubmit->resubmit_num > 1)
1811                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1812                               sizeof(struct rte_vhost_resubmit_desc),
1813                               resubmit_desc_compare);
1814
1815                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1816                 vq->resubmit_inflight = resubmit;
1817         }
1818
1819         return RTE_VHOST_MSG_RESULT_OK;
1820 }
1821
1822 static int
1823 vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,
1824                         int main_fd __rte_unused)
1825 {
1826         struct virtio_net *dev = *pdev;
1827         struct vhost_vring_file file;
1828         struct vhost_virtqueue *vq;
1829         int expected_fds;
1830
1831         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1832         if (validate_msg_fds(msg, expected_fds) != 0)
1833                 return RTE_VHOST_MSG_RESULT_ERR;
1834
1835         file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1836         if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1837                 file.fd = VIRTIO_INVALID_EVENTFD;
1838         else
1839                 file.fd = msg->fds[0];
1840         VHOST_LOG_CONFIG(INFO,
1841                 "vring kick idx:%d file:%d\n", file.index, file.fd);
1842
1843         /* Interpret ring addresses only when ring is started. */
1844         dev = translate_ring_addresses(dev, file.index);
1845         if (!dev)
1846                 return RTE_VHOST_MSG_RESULT_ERR;
1847
1848         *pdev = dev;
1849
1850         vq = dev->virtqueue[file.index];
1851
1852         /*
1853          * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated,
1854          * the ring starts already enabled. Otherwise, it is enabled via
1855          * the SET_VRING_ENABLE message.
1856          */
1857         if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) {
1858                 vq->enabled = 1;
1859                 if (dev->notify_ops->vring_state_changed)
1860                         dev->notify_ops->vring_state_changed(
1861                                 dev->vid, file.index, 1);
1862         }
1863
1864         if (vq->ready) {
1865                 vq->ready = 0;
1866                 vhost_user_notify_queue_state(dev, file.index, 0);
1867         }
1868
1869         if (vq->kickfd >= 0)
1870                 close(vq->kickfd);
1871         vq->kickfd = file.fd;
1872
1873         if (vq_is_packed(dev)) {
1874                 if (vhost_check_queue_inflights_packed(dev, vq)) {
1875                         VHOST_LOG_CONFIG(ERR,
1876                                 "failed to inflights for vq: %d\n", file.index);
1877                         return RTE_VHOST_MSG_RESULT_ERR;
1878                 }
1879         } else {
1880                 if (vhost_check_queue_inflights_split(dev, vq)) {
1881                         VHOST_LOG_CONFIG(ERR,
1882                                 "failed to inflights for vq: %d\n", file.index);
1883                         return RTE_VHOST_MSG_RESULT_ERR;
1884                 }
1885         }
1886
1887         return RTE_VHOST_MSG_RESULT_OK;
1888 }
1889
1890 /*
1891  * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
1892  */
1893 static int
1894 vhost_user_get_vring_base(struct virtio_net **pdev,
1895                         struct VhostUserMsg *msg,
1896                         int main_fd __rte_unused)
1897 {
1898         struct virtio_net *dev = *pdev;
1899         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
1900         uint64_t val;
1901
1902         if (validate_msg_fds(msg, 0) != 0)
1903                 return RTE_VHOST_MSG_RESULT_ERR;
1904
1905         /* We have to stop the queue (virtio) if it is running. */
1906         vhost_destroy_device_notify(dev);
1907
1908         dev->flags &= ~VIRTIO_DEV_READY;
1909         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
1910
1911         /* Here we are safe to get the indexes */
1912         if (vq_is_packed(dev)) {
1913                 /*
1914                  * Bit[0:14]: avail index
1915                  * Bit[15]: avail wrap counter
1916                  */
1917                 val = vq->last_avail_idx & 0x7fff;
1918                 val |= vq->avail_wrap_counter << 15;
1919                 msg->payload.state.num = val;
1920         } else {
1921                 msg->payload.state.num = vq->last_avail_idx;
1922         }
1923
1924         VHOST_LOG_CONFIG(INFO,
1925                 "vring base idx:%d file:%d\n", msg->payload.state.index,
1926                 msg->payload.state.num);
1927         /*
1928          * Based on current qemu vhost-user implementation, this message is
1929          * sent and only sent in vhost_vring_stop.
1930          * TODO: cleanup the vring, it isn't usable since here.
1931          */
1932         if (vq->kickfd >= 0)
1933                 close(vq->kickfd);
1934
1935         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
1936
1937         if (vq->callfd >= 0)
1938                 close(vq->callfd);
1939
1940         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
1941
1942         vq->signalled_used_valid = false;
1943
1944         if (vq_is_packed(dev)) {
1945                 rte_free(vq->shadow_used_packed);
1946                 vq->shadow_used_packed = NULL;
1947         } else {
1948                 rte_free(vq->shadow_used_split);
1949                 vq->shadow_used_split = NULL;
1950                 if (vq->async_pkts_pending)
1951                         rte_free(vq->async_pkts_pending);
1952                 if (vq->async_pkts_info)
1953                         rte_free(vq->async_pkts_info);
1954                 vq->async_pkts_pending = NULL;
1955                 vq->async_pkts_info = NULL;
1956         }
1957
1958         rte_free(vq->batch_copy_elems);
1959         vq->batch_copy_elems = NULL;
1960
1961         msg->size = sizeof(msg->payload.state);
1962         msg->fd_num = 0;
1963
1964         vring_invalidate(dev, vq);
1965
1966         return RTE_VHOST_MSG_RESULT_REPLY;
1967 }
1968
1969 /*
1970  * when virtio queues are ready to work, qemu will send us to
1971  * enable the virtio queue pair.
1972  */
1973 static int
1974 vhost_user_set_vring_enable(struct virtio_net **pdev,
1975                         struct VhostUserMsg *msg,
1976                         int main_fd __rte_unused)
1977 {
1978         struct virtio_net *dev = *pdev;
1979         int enable = (int)msg->payload.state.num;
1980         int index = (int)msg->payload.state.index;
1981
1982         if (validate_msg_fds(msg, 0) != 0)
1983                 return RTE_VHOST_MSG_RESULT_ERR;
1984
1985         VHOST_LOG_CONFIG(INFO,
1986                 "set queue enable: %d to qp idx: %d\n",
1987                 enable, index);
1988
1989         if (enable && dev->virtqueue[index]->async_registered) {
1990                 if (dev->virtqueue[index]->async_pkts_inflight_n) {
1991                         VHOST_LOG_CONFIG(ERR, "failed to enable vring. "
1992                         "async inflight packets must be completed first\n");
1993                         return RTE_VHOST_MSG_RESULT_ERR;
1994                 }
1995         }
1996
1997         dev->virtqueue[index]->enabled = enable;
1998
1999         return RTE_VHOST_MSG_RESULT_OK;
2000 }
2001
2002 static int
2003 vhost_user_get_protocol_features(struct virtio_net **pdev,
2004                         struct VhostUserMsg *msg,
2005                         int main_fd __rte_unused)
2006 {
2007         struct virtio_net *dev = *pdev;
2008         uint64_t features, protocol_features;
2009
2010         if (validate_msg_fds(msg, 0) != 0)
2011                 return RTE_VHOST_MSG_RESULT_ERR;
2012
2013         rte_vhost_driver_get_features(dev->ifname, &features);
2014         rte_vhost_driver_get_protocol_features(dev->ifname, &protocol_features);
2015
2016         msg->payload.u64 = protocol_features;
2017         msg->size = sizeof(msg->payload.u64);
2018         msg->fd_num = 0;
2019
2020         return RTE_VHOST_MSG_RESULT_REPLY;
2021 }
2022
2023 static int
2024 vhost_user_set_protocol_features(struct virtio_net **pdev,
2025                         struct VhostUserMsg *msg,
2026                         int main_fd __rte_unused)
2027 {
2028         struct virtio_net *dev = *pdev;
2029         uint64_t protocol_features = msg->payload.u64;
2030         uint64_t slave_protocol_features = 0;
2031
2032         if (validate_msg_fds(msg, 0) != 0)
2033                 return RTE_VHOST_MSG_RESULT_ERR;
2034
2035         rte_vhost_driver_get_protocol_features(dev->ifname,
2036                         &slave_protocol_features);
2037         if (protocol_features & ~slave_protocol_features) {
2038                 VHOST_LOG_CONFIG(ERR,
2039                         "(%d) received invalid protocol features.\n",
2040                         dev->vid);
2041                 return RTE_VHOST_MSG_RESULT_ERR;
2042         }
2043
2044         dev->protocol_features = protocol_features;
2045         VHOST_LOG_CONFIG(INFO,
2046                 "negotiated Vhost-user protocol features: 0x%" PRIx64 "\n",
2047                 dev->protocol_features);
2048
2049         return RTE_VHOST_MSG_RESULT_OK;
2050 }
2051
2052 static int
2053 vhost_user_set_log_base(struct virtio_net **pdev, struct VhostUserMsg *msg,
2054                         int main_fd __rte_unused)
2055 {
2056         struct virtio_net *dev = *pdev;
2057         int fd = msg->fds[0];
2058         uint64_t size, off;
2059         void *addr;
2060
2061         if (validate_msg_fds(msg, 1) != 0)
2062                 return RTE_VHOST_MSG_RESULT_ERR;
2063
2064         if (fd < 0) {
2065                 VHOST_LOG_CONFIG(ERR, "invalid log fd: %d\n", fd);
2066                 return RTE_VHOST_MSG_RESULT_ERR;
2067         }
2068
2069         if (msg->size != sizeof(VhostUserLog)) {
2070                 VHOST_LOG_CONFIG(ERR,
2071                         "invalid log base msg size: %"PRId32" != %d\n",
2072                         msg->size, (int)sizeof(VhostUserLog));
2073                 return RTE_VHOST_MSG_RESULT_ERR;
2074         }
2075
2076         size = msg->payload.log.mmap_size;
2077         off  = msg->payload.log.mmap_offset;
2078
2079         /* Check for mmap size and offset overflow. */
2080         if (off >= -size) {
2081                 VHOST_LOG_CONFIG(ERR,
2082                         "log offset %#"PRIx64" and log size %#"PRIx64" overflow\n",
2083                         off, size);
2084                 return RTE_VHOST_MSG_RESULT_ERR;
2085         }
2086
2087         VHOST_LOG_CONFIG(INFO,
2088                 "log mmap size: %"PRId64", offset: %"PRId64"\n",
2089                 size, off);
2090
2091         /*
2092          * mmap from 0 to workaround a hugepage mmap bug: mmap will
2093          * fail when offset is not page size aligned.
2094          */
2095         addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
2096         close(fd);
2097         if (addr == MAP_FAILED) {
2098                 VHOST_LOG_CONFIG(ERR, "mmap log base failed!\n");
2099                 return RTE_VHOST_MSG_RESULT_ERR;
2100         }
2101
2102         /*
2103          * Free previously mapped log memory on occasionally
2104          * multiple VHOST_USER_SET_LOG_BASE.
2105          */
2106         if (dev->log_addr) {
2107                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
2108         }
2109         dev->log_addr = (uint64_t)(uintptr_t)addr;
2110         dev->log_base = dev->log_addr + off;
2111         dev->log_size = size;
2112
2113         /*
2114          * The spec is not clear about it (yet), but QEMU doesn't expect
2115          * any payload in the reply.
2116          */
2117         msg->size = 0;
2118         msg->fd_num = 0;
2119
2120         return RTE_VHOST_MSG_RESULT_REPLY;
2121 }
2122
2123 static int vhost_user_set_log_fd(struct virtio_net **pdev __rte_unused,
2124                         struct VhostUserMsg *msg,
2125                         int main_fd __rte_unused)
2126 {
2127         if (validate_msg_fds(msg, 1) != 0)
2128                 return RTE_VHOST_MSG_RESULT_ERR;
2129
2130         close(msg->fds[0]);
2131         VHOST_LOG_CONFIG(INFO, "not implemented.\n");
2132
2133         return RTE_VHOST_MSG_RESULT_OK;
2134 }
2135
2136 /*
2137  * An rarp packet is constructed and broadcasted to notify switches about
2138  * the new location of the migrated VM, so that packets from outside will
2139  * not be lost after migration.
2140  *
2141  * However, we don't actually "send" a rarp packet here, instead, we set
2142  * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
2143  */
2144 static int
2145 vhost_user_send_rarp(struct virtio_net **pdev, struct VhostUserMsg *msg,
2146                         int main_fd __rte_unused)
2147 {
2148         struct virtio_net *dev = *pdev;
2149         uint8_t *mac = (uint8_t *)&msg->payload.u64;
2150         struct rte_vdpa_device *vdpa_dev;
2151
2152         if (validate_msg_fds(msg, 0) != 0)
2153                 return RTE_VHOST_MSG_RESULT_ERR;
2154
2155         VHOST_LOG_CONFIG(DEBUG,
2156                 ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
2157                 mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
2158         memcpy(dev->mac.addr_bytes, mac, 6);
2159
2160         /*
2161          * Set the flag to inject a RARP broadcast packet at
2162          * rte_vhost_dequeue_burst().
2163          *
2164          * __ATOMIC_RELEASE ordering is for making sure the mac is
2165          * copied before the flag is set.
2166          */
2167         __atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
2168         vdpa_dev = dev->vdpa_dev;
2169         if (vdpa_dev && vdpa_dev->ops->migration_done)
2170                 vdpa_dev->ops->migration_done(dev->vid);
2171
2172         return RTE_VHOST_MSG_RESULT_OK;
2173 }
2174
2175 static int
2176 vhost_user_net_set_mtu(struct virtio_net **pdev, struct VhostUserMsg *msg,
2177                         int main_fd __rte_unused)
2178 {
2179         struct virtio_net *dev = *pdev;
2180
2181         if (validate_msg_fds(msg, 0) != 0)
2182                 return RTE_VHOST_MSG_RESULT_ERR;
2183
2184         if (msg->payload.u64 < VIRTIO_MIN_MTU ||
2185                         msg->payload.u64 > VIRTIO_MAX_MTU) {
2186                 VHOST_LOG_CONFIG(ERR, "Invalid MTU size (%"PRIu64")\n",
2187                                 msg->payload.u64);
2188
2189                 return RTE_VHOST_MSG_RESULT_ERR;
2190         }
2191
2192         dev->mtu = msg->payload.u64;
2193
2194         return RTE_VHOST_MSG_RESULT_OK;
2195 }
2196
2197 static int
2198 vhost_user_set_req_fd(struct virtio_net **pdev, struct VhostUserMsg *msg,
2199                         int main_fd __rte_unused)
2200 {
2201         struct virtio_net *dev = *pdev;
2202         int fd = msg->fds[0];
2203
2204         if (validate_msg_fds(msg, 1) != 0)
2205                 return RTE_VHOST_MSG_RESULT_ERR;
2206
2207         if (fd < 0) {
2208                 VHOST_LOG_CONFIG(ERR,
2209                                 "Invalid file descriptor for slave channel (%d)\n",
2210                                 fd);
2211                 return RTE_VHOST_MSG_RESULT_ERR;
2212         }
2213
2214         if (dev->slave_req_fd >= 0)
2215                 close(dev->slave_req_fd);
2216
2217         dev->slave_req_fd = fd;
2218
2219         return RTE_VHOST_MSG_RESULT_OK;
2220 }
2221
2222 static int
2223 is_vring_iotlb_split(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2224 {
2225         struct vhost_vring_addr *ra;
2226         uint64_t start, end, len;
2227
2228         start = imsg->iova;
2229         end = start + imsg->size;
2230
2231         ra = &vq->ring_addrs;
2232         len = sizeof(struct vring_desc) * vq->size;
2233         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2234                 return 1;
2235
2236         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
2237         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2238                 return 1;
2239
2240         len = sizeof(struct vring_used) +
2241                sizeof(struct vring_used_elem) * vq->size;
2242         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2243                 return 1;
2244
2245         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2246                 len = sizeof(uint64_t);
2247                 if (ra->log_guest_addr < end &&
2248                     (ra->log_guest_addr + len) > start)
2249                         return 1;
2250         }
2251
2252         return 0;
2253 }
2254
2255 static int
2256 is_vring_iotlb_packed(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2257 {
2258         struct vhost_vring_addr *ra;
2259         uint64_t start, end, len;
2260
2261         start = imsg->iova;
2262         end = start + imsg->size;
2263
2264         ra = &vq->ring_addrs;
2265         len = sizeof(struct vring_packed_desc) * vq->size;
2266         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2267                 return 1;
2268
2269         len = sizeof(struct vring_packed_desc_event);
2270         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2271                 return 1;
2272
2273         len = sizeof(struct vring_packed_desc_event);
2274         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2275                 return 1;
2276
2277         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2278                 len = sizeof(uint64_t);
2279                 if (ra->log_guest_addr < end &&
2280                     (ra->log_guest_addr + len) > start)
2281                         return 1;
2282         }
2283
2284         return 0;
2285 }
2286
2287 static int is_vring_iotlb(struct virtio_net *dev,
2288                           struct vhost_virtqueue *vq,
2289                           struct vhost_iotlb_msg *imsg)
2290 {
2291         if (vq_is_packed(dev))
2292                 return is_vring_iotlb_packed(vq, imsg);
2293         else
2294                 return is_vring_iotlb_split(vq, imsg);
2295 }
2296
2297 static int
2298 vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg,
2299                         int main_fd __rte_unused)
2300 {
2301         struct virtio_net *dev = *pdev;
2302         struct vhost_iotlb_msg *imsg = &msg->payload.iotlb;
2303         uint16_t i;
2304         uint64_t vva, len;
2305
2306         if (validate_msg_fds(msg, 0) != 0)
2307                 return RTE_VHOST_MSG_RESULT_ERR;
2308
2309         switch (imsg->type) {
2310         case VHOST_IOTLB_UPDATE:
2311                 len = imsg->size;
2312                 vva = qva_to_vva(dev, imsg->uaddr, &len);
2313                 if (!vva)
2314                         return RTE_VHOST_MSG_RESULT_ERR;
2315
2316                 for (i = 0; i < dev->nr_vring; i++) {
2317                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2318
2319                         if (!vq)
2320                                 continue;
2321
2322                         vhost_user_iotlb_cache_insert(vq, imsg->iova, vva,
2323                                         len, imsg->perm);
2324
2325                         if (is_vring_iotlb(dev, vq, imsg))
2326                                 *pdev = dev = translate_ring_addresses(dev, i);
2327                 }
2328                 break;
2329         case VHOST_IOTLB_INVALIDATE:
2330                 for (i = 0; i < dev->nr_vring; i++) {
2331                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2332
2333                         if (!vq)
2334                                 continue;
2335
2336                         vhost_user_iotlb_cache_remove(vq, imsg->iova,
2337                                         imsg->size);
2338
2339                         if (is_vring_iotlb(dev, vq, imsg))
2340                                 vring_invalidate(dev, vq);
2341                 }
2342                 break;
2343         default:
2344                 VHOST_LOG_CONFIG(ERR, "Invalid IOTLB message type (%d)\n",
2345                                 imsg->type);
2346                 return RTE_VHOST_MSG_RESULT_ERR;
2347         }
2348
2349         return RTE_VHOST_MSG_RESULT_OK;
2350 }
2351
2352 static int
2353 vhost_user_set_postcopy_advise(struct virtio_net **pdev,
2354                         struct VhostUserMsg *msg,
2355                         int main_fd __rte_unused)
2356 {
2357         struct virtio_net *dev = *pdev;
2358 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
2359         struct uffdio_api api_struct;
2360
2361         if (validate_msg_fds(msg, 0) != 0)
2362                 return RTE_VHOST_MSG_RESULT_ERR;
2363
2364         dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
2365
2366         if (dev->postcopy_ufd == -1) {
2367                 VHOST_LOG_CONFIG(ERR, "Userfaultfd not available: %s\n",
2368                         strerror(errno));
2369                 return RTE_VHOST_MSG_RESULT_ERR;
2370         }
2371         api_struct.api = UFFD_API;
2372         api_struct.features = 0;
2373         if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
2374                 VHOST_LOG_CONFIG(ERR, "UFFDIO_API ioctl failure: %s\n",
2375                         strerror(errno));
2376                 close(dev->postcopy_ufd);
2377                 dev->postcopy_ufd = -1;
2378                 return RTE_VHOST_MSG_RESULT_ERR;
2379         }
2380         msg->fds[0] = dev->postcopy_ufd;
2381         msg->fd_num = 1;
2382
2383         return RTE_VHOST_MSG_RESULT_REPLY;
2384 #else
2385         dev->postcopy_ufd = -1;
2386         msg->fd_num = 0;
2387
2388         return RTE_VHOST_MSG_RESULT_ERR;
2389 #endif
2390 }
2391
2392 static int
2393 vhost_user_set_postcopy_listen(struct virtio_net **pdev,
2394                         struct VhostUserMsg *msg __rte_unused,
2395                         int main_fd __rte_unused)
2396 {
2397         struct virtio_net *dev = *pdev;
2398
2399         if (validate_msg_fds(msg, 0) != 0)
2400                 return RTE_VHOST_MSG_RESULT_ERR;
2401
2402         if (dev->mem && dev->mem->nregions) {
2403                 VHOST_LOG_CONFIG(ERR,
2404                         "Regions already registered at postcopy-listen\n");
2405                 return RTE_VHOST_MSG_RESULT_ERR;
2406         }
2407         dev->postcopy_listening = 1;
2408
2409         return RTE_VHOST_MSG_RESULT_OK;
2410 }
2411
2412 static int
2413 vhost_user_postcopy_end(struct virtio_net **pdev, struct VhostUserMsg *msg,
2414                         int main_fd __rte_unused)
2415 {
2416         struct virtio_net *dev = *pdev;
2417
2418         if (validate_msg_fds(msg, 0) != 0)
2419                 return RTE_VHOST_MSG_RESULT_ERR;
2420
2421         dev->postcopy_listening = 0;
2422         if (dev->postcopy_ufd >= 0) {
2423                 close(dev->postcopy_ufd);
2424                 dev->postcopy_ufd = -1;
2425         }
2426
2427         msg->payload.u64 = 0;
2428         msg->size = sizeof(msg->payload.u64);
2429         msg->fd_num = 0;
2430
2431         return RTE_VHOST_MSG_RESULT_REPLY;
2432 }
2433
2434 static int
2435 vhost_user_get_status(struct virtio_net **pdev, struct VhostUserMsg *msg,
2436                       int main_fd __rte_unused)
2437 {
2438         struct virtio_net *dev = *pdev;
2439
2440         if (validate_msg_fds(msg, 0) != 0)
2441                 return RTE_VHOST_MSG_RESULT_ERR;
2442
2443         msg->payload.u64 = dev->status;
2444         msg->size = sizeof(msg->payload.u64);
2445         msg->fd_num = 0;
2446
2447         return RTE_VHOST_MSG_RESULT_REPLY;
2448 }
2449
2450 static int
2451 vhost_user_set_status(struct virtio_net **pdev, struct VhostUserMsg *msg,
2452                         int main_fd __rte_unused)
2453 {
2454         struct virtio_net *dev = *pdev;
2455
2456         if (validate_msg_fds(msg, 0) != 0)
2457                 return RTE_VHOST_MSG_RESULT_ERR;
2458
2459         /* As per Virtio specification, the device status is 8bits long */
2460         if (msg->payload.u64 > UINT8_MAX) {
2461                 VHOST_LOG_CONFIG(ERR, "Invalid VHOST_USER_SET_STATUS payload 0x%" PRIx64 "\n",
2462                                 msg->payload.u64);
2463                 return RTE_VHOST_MSG_RESULT_ERR;
2464         }
2465
2466         dev->status = msg->payload.u64;
2467
2468         if ((dev->status & VIRTIO_DEVICE_STATUS_FEATURES_OK) &&
2469             (dev->flags & VIRTIO_DEV_FEATURES_FAILED)) {
2470                 VHOST_LOG_CONFIG(ERR, "FEATURES_OK bit is set but feature negotiation failed\n");
2471                 /*
2472                  * Clear the bit to let the driver know about the feature
2473                  * negotiation failure
2474                  */
2475                 dev->status &= ~VIRTIO_DEVICE_STATUS_FEATURES_OK;
2476         }
2477
2478         VHOST_LOG_CONFIG(INFO, "New device status(0x%08x):\n"
2479                         "\t-RESET: %u\n"
2480                         "\t-ACKNOWLEDGE: %u\n"
2481                         "\t-DRIVER: %u\n"
2482                         "\t-FEATURES_OK: %u\n"
2483                         "\t-DRIVER_OK: %u\n"
2484                         "\t-DEVICE_NEED_RESET: %u\n"
2485                         "\t-FAILED: %u\n",
2486                         dev->status,
2487                         (dev->status == VIRTIO_DEVICE_STATUS_RESET),
2488                         !!(dev->status & VIRTIO_DEVICE_STATUS_ACK),
2489                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER),
2490                         !!(dev->status & VIRTIO_DEVICE_STATUS_FEATURES_OK),
2491                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK),
2492                         !!(dev->status & VIRTIO_DEVICE_STATUS_DEV_NEED_RESET),
2493                         !!(dev->status & VIRTIO_DEVICE_STATUS_FAILED));
2494
2495         return RTE_VHOST_MSG_RESULT_OK;
2496 }
2497
2498 typedef int (*vhost_message_handler_t)(struct virtio_net **pdev,
2499                                         struct VhostUserMsg *msg,
2500                                         int main_fd);
2501 static vhost_message_handler_t vhost_message_handlers[VHOST_USER_MAX] = {
2502         [VHOST_USER_NONE] = NULL,
2503         [VHOST_USER_GET_FEATURES] = vhost_user_get_features,
2504         [VHOST_USER_SET_FEATURES] = vhost_user_set_features,
2505         [VHOST_USER_SET_OWNER] = vhost_user_set_owner,
2506         [VHOST_USER_RESET_OWNER] = vhost_user_reset_owner,
2507         [VHOST_USER_SET_MEM_TABLE] = vhost_user_set_mem_table,
2508         [VHOST_USER_SET_LOG_BASE] = vhost_user_set_log_base,
2509         [VHOST_USER_SET_LOG_FD] = vhost_user_set_log_fd,
2510         [VHOST_USER_SET_VRING_NUM] = vhost_user_set_vring_num,
2511         [VHOST_USER_SET_VRING_ADDR] = vhost_user_set_vring_addr,
2512         [VHOST_USER_SET_VRING_BASE] = vhost_user_set_vring_base,
2513         [VHOST_USER_GET_VRING_BASE] = vhost_user_get_vring_base,
2514         [VHOST_USER_SET_VRING_KICK] = vhost_user_set_vring_kick,
2515         [VHOST_USER_SET_VRING_CALL] = vhost_user_set_vring_call,
2516         [VHOST_USER_SET_VRING_ERR] = vhost_user_set_vring_err,
2517         [VHOST_USER_GET_PROTOCOL_FEATURES] = vhost_user_get_protocol_features,
2518         [VHOST_USER_SET_PROTOCOL_FEATURES] = vhost_user_set_protocol_features,
2519         [VHOST_USER_GET_QUEUE_NUM] = vhost_user_get_queue_num,
2520         [VHOST_USER_SET_VRING_ENABLE] = vhost_user_set_vring_enable,
2521         [VHOST_USER_SEND_RARP] = vhost_user_send_rarp,
2522         [VHOST_USER_NET_SET_MTU] = vhost_user_net_set_mtu,
2523         [VHOST_USER_SET_SLAVE_REQ_FD] = vhost_user_set_req_fd,
2524         [VHOST_USER_IOTLB_MSG] = vhost_user_iotlb_msg,
2525         [VHOST_USER_POSTCOPY_ADVISE] = vhost_user_set_postcopy_advise,
2526         [VHOST_USER_POSTCOPY_LISTEN] = vhost_user_set_postcopy_listen,
2527         [VHOST_USER_POSTCOPY_END] = vhost_user_postcopy_end,
2528         [VHOST_USER_GET_INFLIGHT_FD] = vhost_user_get_inflight_fd,
2529         [VHOST_USER_SET_INFLIGHT_FD] = vhost_user_set_inflight_fd,
2530         [VHOST_USER_SET_STATUS] = vhost_user_set_status,
2531         [VHOST_USER_GET_STATUS] = vhost_user_get_status,
2532 };
2533
2534 /* return bytes# of read on success or negative val on failure. */
2535 static int
2536 read_vhost_message(int sockfd, struct VhostUserMsg *msg)
2537 {
2538         int ret;
2539
2540         ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
2541                 msg->fds, VHOST_MEMORY_MAX_NREGIONS, &msg->fd_num);
2542         if (ret <= 0) {
2543                 return ret;
2544         } else if (ret != VHOST_USER_HDR_SIZE) {
2545                 VHOST_LOG_CONFIG(ERR, "Unexpected header size read\n");
2546                 close_msg_fds(msg);
2547                 return -1;
2548         }
2549
2550         if (msg->size) {
2551                 if (msg->size > sizeof(msg->payload)) {
2552                         VHOST_LOG_CONFIG(ERR,
2553                                 "invalid msg size: %d\n", msg->size);
2554                         return -1;
2555                 }
2556                 ret = read(sockfd, &msg->payload, msg->size);
2557                 if (ret <= 0)
2558                         return ret;
2559                 if (ret != (int)msg->size) {
2560                         VHOST_LOG_CONFIG(ERR,
2561                                 "read control message failed\n");
2562                         return -1;
2563                 }
2564         }
2565
2566         return ret;
2567 }
2568
2569 static int
2570 send_vhost_message(int sockfd, struct VhostUserMsg *msg)
2571 {
2572         if (!msg)
2573                 return 0;
2574
2575         return send_fd_message(sockfd, (char *)msg,
2576                 VHOST_USER_HDR_SIZE + msg->size, msg->fds, msg->fd_num);
2577 }
2578
2579 static int
2580 send_vhost_reply(int sockfd, struct VhostUserMsg *msg)
2581 {
2582         if (!msg)
2583                 return 0;
2584
2585         msg->flags &= ~VHOST_USER_VERSION_MASK;
2586         msg->flags &= ~VHOST_USER_NEED_REPLY;
2587         msg->flags |= VHOST_USER_VERSION;
2588         msg->flags |= VHOST_USER_REPLY_MASK;
2589
2590         return send_vhost_message(sockfd, msg);
2591 }
2592
2593 static int
2594 send_vhost_slave_message(struct virtio_net *dev, struct VhostUserMsg *msg)
2595 {
2596         int ret;
2597
2598         if (msg->flags & VHOST_USER_NEED_REPLY)
2599                 rte_spinlock_lock(&dev->slave_req_lock);
2600
2601         ret = send_vhost_message(dev->slave_req_fd, msg);
2602         if (ret < 0 && (msg->flags & VHOST_USER_NEED_REPLY))
2603                 rte_spinlock_unlock(&dev->slave_req_lock);
2604
2605         return ret;
2606 }
2607
2608 /*
2609  * Allocate a queue pair if it hasn't been allocated yet
2610  */
2611 static int
2612 vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev,
2613                         struct VhostUserMsg *msg)
2614 {
2615         uint32_t vring_idx;
2616
2617         switch (msg->request.master) {
2618         case VHOST_USER_SET_VRING_KICK:
2619         case VHOST_USER_SET_VRING_CALL:
2620         case VHOST_USER_SET_VRING_ERR:
2621                 vring_idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
2622                 break;
2623         case VHOST_USER_SET_VRING_NUM:
2624         case VHOST_USER_SET_VRING_BASE:
2625         case VHOST_USER_SET_VRING_ENABLE:
2626                 vring_idx = msg->payload.state.index;
2627                 break;
2628         case VHOST_USER_SET_VRING_ADDR:
2629                 vring_idx = msg->payload.addr.index;
2630                 break;
2631         default:
2632                 return 0;
2633         }
2634
2635         if (vring_idx >= VHOST_MAX_VRING) {
2636                 VHOST_LOG_CONFIG(ERR,
2637                         "invalid vring index: %u\n", vring_idx);
2638                 return -1;
2639         }
2640
2641         if (dev->virtqueue[vring_idx])
2642                 return 0;
2643
2644         return alloc_vring_queue(dev, vring_idx);
2645 }
2646
2647 static void
2648 vhost_user_lock_all_queue_pairs(struct virtio_net *dev)
2649 {
2650         unsigned int i = 0;
2651         unsigned int vq_num = 0;
2652
2653         while (vq_num < dev->nr_vring) {
2654                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2655
2656                 if (vq) {
2657                         rte_spinlock_lock(&vq->access_lock);
2658                         vq_num++;
2659                 }
2660                 i++;
2661         }
2662 }
2663
2664 static void
2665 vhost_user_unlock_all_queue_pairs(struct virtio_net *dev)
2666 {
2667         unsigned int i = 0;
2668         unsigned int vq_num = 0;
2669
2670         while (vq_num < dev->nr_vring) {
2671                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2672
2673                 if (vq) {
2674                         rte_spinlock_unlock(&vq->access_lock);
2675                         vq_num++;
2676                 }
2677                 i++;
2678         }
2679 }
2680
2681 int
2682 vhost_user_msg_handler(int vid, int fd)
2683 {
2684         struct virtio_net *dev;
2685         struct VhostUserMsg msg;
2686         struct rte_vdpa_device *vdpa_dev;
2687         int ret;
2688         int unlock_required = 0;
2689         bool handled;
2690         int request;
2691         uint32_t i;
2692
2693         dev = get_device(vid);
2694         if (dev == NULL)
2695                 return -1;
2696
2697         if (!dev->notify_ops) {
2698                 dev->notify_ops = vhost_driver_callback_get(dev->ifname);
2699                 if (!dev->notify_ops) {
2700                         VHOST_LOG_CONFIG(ERR,
2701                                 "failed to get callback ops for driver %s\n",
2702                                 dev->ifname);
2703                         return -1;
2704                 }
2705         }
2706
2707         ret = read_vhost_message(fd, &msg);
2708         if (ret <= 0) {
2709                 if (ret < 0)
2710                         VHOST_LOG_CONFIG(ERR,
2711                                 "vhost read message failed\n");
2712                 else
2713                         VHOST_LOG_CONFIG(INFO,
2714                                 "vhost peer closed\n");
2715
2716                 return -1;
2717         }
2718
2719         ret = 0;
2720         request = msg.request.master;
2721         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX &&
2722                         vhost_message_str[request]) {
2723                 if (request != VHOST_USER_IOTLB_MSG)
2724                         VHOST_LOG_CONFIG(INFO, "read message %s\n",
2725                                 vhost_message_str[request]);
2726                 else
2727                         VHOST_LOG_CONFIG(DEBUG, "read message %s\n",
2728                                 vhost_message_str[request]);
2729         } else {
2730                 VHOST_LOG_CONFIG(DEBUG, "External request %d\n", request);
2731         }
2732
2733         ret = vhost_user_check_and_alloc_queue_pair(dev, &msg);
2734         if (ret < 0) {
2735                 VHOST_LOG_CONFIG(ERR,
2736                         "failed to alloc queue\n");
2737                 return -1;
2738         }
2739
2740         /*
2741          * Note: we don't lock all queues on VHOST_USER_GET_VRING_BASE
2742          * and VHOST_USER_RESET_OWNER, since it is sent when virtio stops
2743          * and device is destroyed. destroy_device waits for queues to be
2744          * inactive, so it is safe. Otherwise taking the access_lock
2745          * would cause a dead lock.
2746          */
2747         switch (request) {
2748         case VHOST_USER_SET_FEATURES:
2749         case VHOST_USER_SET_PROTOCOL_FEATURES:
2750         case VHOST_USER_SET_OWNER:
2751         case VHOST_USER_SET_MEM_TABLE:
2752         case VHOST_USER_SET_LOG_BASE:
2753         case VHOST_USER_SET_LOG_FD:
2754         case VHOST_USER_SET_VRING_NUM:
2755         case VHOST_USER_SET_VRING_ADDR:
2756         case VHOST_USER_SET_VRING_BASE:
2757         case VHOST_USER_SET_VRING_KICK:
2758         case VHOST_USER_SET_VRING_CALL:
2759         case VHOST_USER_SET_VRING_ERR:
2760         case VHOST_USER_SET_VRING_ENABLE:
2761         case VHOST_USER_SEND_RARP:
2762         case VHOST_USER_NET_SET_MTU:
2763         case VHOST_USER_SET_SLAVE_REQ_FD:
2764                 if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
2765                         vhost_user_lock_all_queue_pairs(dev);
2766                         unlock_required = 1;
2767                 }
2768                 break;
2769         default:
2770                 break;
2771
2772         }
2773
2774         handled = false;
2775         if (dev->extern_ops.pre_msg_handle) {
2776                 ret = (*dev->extern_ops.pre_msg_handle)(dev->vid,
2777                                 (void *)&msg);
2778                 switch (ret) {
2779                 case RTE_VHOST_MSG_RESULT_REPLY:
2780                         send_vhost_reply(fd, &msg);
2781                         /* Fall-through */
2782                 case RTE_VHOST_MSG_RESULT_ERR:
2783                 case RTE_VHOST_MSG_RESULT_OK:
2784                         handled = true;
2785                         goto skip_to_post_handle;
2786                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
2787                 default:
2788                         break;
2789                 }
2790         }
2791
2792         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX) {
2793                 if (!vhost_message_handlers[request])
2794                         goto skip_to_post_handle;
2795                 ret = vhost_message_handlers[request](&dev, &msg, fd);
2796
2797                 switch (ret) {
2798                 case RTE_VHOST_MSG_RESULT_ERR:
2799                         VHOST_LOG_CONFIG(ERR,
2800                                 "Processing %s failed.\n",
2801                                 vhost_message_str[request]);
2802                         handled = true;
2803                         break;
2804                 case RTE_VHOST_MSG_RESULT_OK:
2805                         VHOST_LOG_CONFIG(DEBUG,
2806                                 "Processing %s succeeded.\n",
2807                                 vhost_message_str[request]);
2808                         handled = true;
2809                         break;
2810                 case RTE_VHOST_MSG_RESULT_REPLY:
2811                         VHOST_LOG_CONFIG(DEBUG,
2812                                 "Processing %s succeeded and needs reply.\n",
2813                                 vhost_message_str[request]);
2814                         send_vhost_reply(fd, &msg);
2815                         handled = true;
2816                         break;
2817                 default:
2818                         break;
2819                 }
2820         }
2821
2822 skip_to_post_handle:
2823         if (ret != RTE_VHOST_MSG_RESULT_ERR &&
2824                         dev->extern_ops.post_msg_handle) {
2825                 ret = (*dev->extern_ops.post_msg_handle)(dev->vid,
2826                                 (void *)&msg);
2827                 switch (ret) {
2828                 case RTE_VHOST_MSG_RESULT_REPLY:
2829                         send_vhost_reply(fd, &msg);
2830                         /* Fall-through */
2831                 case RTE_VHOST_MSG_RESULT_ERR:
2832                 case RTE_VHOST_MSG_RESULT_OK:
2833                         handled = true;
2834                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
2835                 default:
2836                         break;
2837                 }
2838         }
2839
2840         if (unlock_required)
2841                 vhost_user_unlock_all_queue_pairs(dev);
2842
2843         /* If message was not handled at this stage, treat it as an error */
2844         if (!handled) {
2845                 VHOST_LOG_CONFIG(ERR,
2846                         "vhost message (req: %d) was not handled.\n", request);
2847                 close_msg_fds(&msg);
2848                 ret = RTE_VHOST_MSG_RESULT_ERR;
2849         }
2850
2851         /*
2852          * If the request required a reply that was already sent,
2853          * this optional reply-ack won't be sent as the
2854          * VHOST_USER_NEED_REPLY was cleared in send_vhost_reply().
2855          */
2856         if (msg.flags & VHOST_USER_NEED_REPLY) {
2857                 msg.payload.u64 = ret == RTE_VHOST_MSG_RESULT_ERR;
2858                 msg.size = sizeof(msg.payload.u64);
2859                 msg.fd_num = 0;
2860                 send_vhost_reply(fd, &msg);
2861         } else if (ret == RTE_VHOST_MSG_RESULT_ERR) {
2862                 VHOST_LOG_CONFIG(ERR,
2863                         "vhost message handling failed.\n");
2864                 return -1;
2865         }
2866
2867         for (i = 0; i < dev->nr_vring; i++) {
2868                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2869                 bool cur_ready = vq_is_ready(dev, vq);
2870
2871                 if (cur_ready != (vq && vq->ready)) {
2872                         vq->ready = cur_ready;
2873                         vhost_user_notify_queue_state(dev, i, cur_ready);
2874                 }
2875         }
2876
2877
2878         if (!virtio_is_ready(dev))
2879                 goto out;
2880
2881         /*
2882          * Virtio is now ready. If not done already, it is time
2883          * to notify the application it can process the rings and
2884          * configure the vDPA device if present.
2885          */
2886
2887         if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
2888                 if (dev->notify_ops->new_device(dev->vid) == 0)
2889                         dev->flags |= VIRTIO_DEV_RUNNING;
2890         }
2891
2892         vdpa_dev = dev->vdpa_dev;
2893         if (!vdpa_dev)
2894                 goto out;
2895
2896         if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
2897                 if (vdpa_dev->ops->dev_conf(dev->vid))
2898                         VHOST_LOG_CONFIG(ERR,
2899                                          "Failed to configure vDPA device\n");
2900                 else
2901                         dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED;
2902         }
2903
2904 out:
2905         return 0;
2906 }
2907
2908 static int process_slave_message_reply(struct virtio_net *dev,
2909                                        const struct VhostUserMsg *msg)
2910 {
2911         struct VhostUserMsg msg_reply;
2912         int ret;
2913
2914         if ((msg->flags & VHOST_USER_NEED_REPLY) == 0)
2915                 return 0;
2916
2917         ret = read_vhost_message(dev->slave_req_fd, &msg_reply);
2918         if (ret <= 0) {
2919                 if (ret < 0)
2920                         VHOST_LOG_CONFIG(ERR,
2921                                 "vhost read slave message reply failed\n");
2922                 else
2923                         VHOST_LOG_CONFIG(INFO,
2924                                 "vhost peer closed\n");
2925                 ret = -1;
2926                 goto out;
2927         }
2928
2929         ret = 0;
2930         if (msg_reply.request.slave != msg->request.slave) {
2931                 VHOST_LOG_CONFIG(ERR,
2932                         "Received unexpected msg type (%u), expected %u\n",
2933                         msg_reply.request.slave, msg->request.slave);
2934                 ret = -1;
2935                 goto out;
2936         }
2937
2938         ret = msg_reply.payload.u64 ? -1 : 0;
2939
2940 out:
2941         rte_spinlock_unlock(&dev->slave_req_lock);
2942         return ret;
2943 }
2944
2945 int
2946 vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
2947 {
2948         int ret;
2949         struct VhostUserMsg msg = {
2950                 .request.slave = VHOST_USER_SLAVE_IOTLB_MSG,
2951                 .flags = VHOST_USER_VERSION,
2952                 .size = sizeof(msg.payload.iotlb),
2953                 .payload.iotlb = {
2954                         .iova = iova,
2955                         .perm = perm,
2956                         .type = VHOST_IOTLB_MISS,
2957                 },
2958         };
2959
2960         ret = send_vhost_message(dev->slave_req_fd, &msg);
2961         if (ret < 0) {
2962                 VHOST_LOG_CONFIG(ERR,
2963                                 "Failed to send IOTLB miss message (%d)\n",
2964                                 ret);
2965                 return ret;
2966         }
2967
2968         return 0;
2969 }
2970
2971 static int
2972 vhost_user_slave_config_change(struct virtio_net *dev, bool need_reply)
2973 {
2974         int ret;
2975         struct VhostUserMsg msg = {
2976                 .request.slave = VHOST_USER_SLAVE_CONFIG_CHANGE_MSG,
2977                 .flags = VHOST_USER_VERSION,
2978                 .size = 0,
2979         };
2980
2981         if (need_reply)
2982                 msg.flags |= VHOST_USER_NEED_REPLY;
2983
2984         ret = send_vhost_slave_message(dev, &msg);
2985         if (ret < 0) {
2986                 VHOST_LOG_CONFIG(ERR,
2987                                 "Failed to send config change (%d)\n",
2988                                 ret);
2989                 return ret;
2990         }
2991
2992         return process_slave_message_reply(dev, &msg);
2993 }
2994
2995 int
2996 rte_vhost_slave_config_change(int vid, bool need_reply)
2997 {
2998         struct virtio_net *dev;
2999
3000         dev = get_device(vid);
3001         if (!dev)
3002                 return -ENODEV;
3003
3004         return vhost_user_slave_config_change(dev, need_reply);
3005 }
3006
3007 static int vhost_user_slave_set_vring_host_notifier(struct virtio_net *dev,
3008                                                     int index, int fd,
3009                                                     uint64_t offset,
3010                                                     uint64_t size)
3011 {
3012         int ret;
3013         struct VhostUserMsg msg = {
3014                 .request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
3015                 .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY,
3016                 .size = sizeof(msg.payload.area),
3017                 .payload.area = {
3018                         .u64 = index & VHOST_USER_VRING_IDX_MASK,
3019                         .size = size,
3020                         .offset = offset,
3021                 },
3022         };
3023
3024         if (fd < 0)
3025                 msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
3026         else {
3027                 msg.fds[0] = fd;
3028                 msg.fd_num = 1;
3029         }
3030
3031         ret = send_vhost_slave_message(dev, &msg);
3032         if (ret < 0) {
3033                 VHOST_LOG_CONFIG(ERR,
3034                         "Failed to set host notifier (%d)\n", ret);
3035                 return ret;
3036         }
3037
3038         return process_slave_message_reply(dev, &msg);
3039 }
3040
3041 int rte_vhost_host_notifier_ctrl(int vid, uint16_t qid, bool enable)
3042 {
3043         struct virtio_net *dev;
3044         struct rte_vdpa_device *vdpa_dev;
3045         int vfio_device_fd, ret = 0;
3046         uint64_t offset, size;
3047         unsigned int i, q_start, q_last;
3048
3049         dev = get_device(vid);
3050         if (!dev)
3051                 return -ENODEV;
3052
3053         vdpa_dev = dev->vdpa_dev;
3054         if (vdpa_dev == NULL)
3055                 return -ENODEV;
3056
3057         if (!(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
3058             !(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) ||
3059             !(dev->protocol_features &
3060                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) ||
3061             !(dev->protocol_features &
3062                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD)) ||
3063             !(dev->protocol_features &
3064                         (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER)))
3065                 return -ENOTSUP;
3066
3067         if (qid == RTE_VHOST_QUEUE_ALL) {
3068                 q_start = 0;
3069                 q_last = dev->nr_vring - 1;
3070         } else {
3071                 if (qid >= dev->nr_vring)
3072                         return -EINVAL;
3073                 q_start = qid;
3074                 q_last = qid;
3075         }
3076
3077         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_vfio_device_fd, -ENOTSUP);
3078         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_notify_area, -ENOTSUP);
3079
3080         vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid);
3081         if (vfio_device_fd < 0)
3082                 return -ENOTSUP;
3083
3084         if (enable) {
3085                 for (i = q_start; i <= q_last; i++) {
3086                         if (vdpa_dev->ops->get_notify_area(vid, i, &offset,
3087                                         &size) < 0) {
3088                                 ret = -ENOTSUP;
3089                                 goto disable;
3090                         }
3091
3092                         if (vhost_user_slave_set_vring_host_notifier(dev, i,
3093                                         vfio_device_fd, offset, size) < 0) {
3094                                 ret = -EFAULT;
3095                                 goto disable;
3096                         }
3097                 }
3098         } else {
3099 disable:
3100                 for (i = q_start; i <= q_last; i++) {
3101                         vhost_user_slave_set_vring_host_notifier(dev, i, -1,
3102                                         0, 0);
3103                 }
3104         }
3105
3106         return ret;
3107 }