lib/librte_vhost/vhost_user.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 /* Security model
   6  * --------------
   7  * The vhost-user protocol connection is an external interface, so it must be
   8  * robust against invalid inputs.
   9  *
  10  * This is important because the vhost-user master is only one step removed
  11  * from the guest.  Malicious guests that have escaped will then launch further
  12  * attacks from the vhost-user master.
  13  *
  14  * Even in deployments where guests are trusted, a bug in the vhost-user master
  15  * can still cause invalid messages to be sent.  Such messages must not
  16  * compromise the stability of the DPDK application by causing crashes, memory
  17  * corruption, or other problematic behavior.
  18  *
  19  * Do not assume received VhostUserMsg fields contain sensible values!
  20  */
  21
  22 #include <stdint.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28 #include <sys/ioctl.h>
  29 #include <sys/mman.h>
  30 #include <sys/types.h>
  31 #include <sys/stat.h>
  32 #include <sys/syscall.h>
  33 #include <assert.h>
  34 #ifdef RTE_LIBRTE_VHOST_NUMA
  35 #include <numaif.h>
  36 #endif
  37 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
  38 #include <linux/userfaultfd.h>
  39 #endif
  40 #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
  41 #include <linux/memfd.h>
  42 #define MEMFD_SUPPORTED
  43 #endif
  44
  45 #include <rte_common.h>
  46 #include <rte_malloc.h>
  47 #include <rte_log.h>
  48
  49 #include "iotlb.h"
  50 #include "vhost.h"
  51 #include "vhost_user.h"
  52
  53 #define VIRTIO_MIN_MTU 68
  54 #define VIRTIO_MAX_MTU 65535
  55
  56 #define INFLIGHT_ALIGNMENT      64
  57 #define INFLIGHT_VERSION        0x1
  58
  59 static const char *vhost_message_str[VHOST_USER_MAX] = {
  60         [VHOST_USER_NONE] = "VHOST_USER_NONE",
  61         [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
  62         [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
  63         [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
  64         [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
  65         [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
  66         [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
  67         [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
  68         [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
  69         [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
  70         [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
  71         [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
  72         [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
  73         [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
  74         [VHOST_USER_SET_VRING_ERR]  = "VHOST_USER_SET_VRING_ERR",
  75         [VHOST_USER_GET_PROTOCOL_FEATURES]  = "VHOST_USER_GET_PROTOCOL_FEATURES",
  76         [VHOST_USER_SET_PROTOCOL_FEATURES]  = "VHOST_USER_SET_PROTOCOL_FEATURES",
  77         [VHOST_USER_GET_QUEUE_NUM]  = "VHOST_USER_GET_QUEUE_NUM",
  78         [VHOST_USER_SET_VRING_ENABLE]  = "VHOST_USER_SET_VRING_ENABLE",
  79         [VHOST_USER_SEND_RARP]  = "VHOST_USER_SEND_RARP",
  80         [VHOST_USER_NET_SET_MTU]  = "VHOST_USER_NET_SET_MTU",
  81         [VHOST_USER_SET_SLAVE_REQ_FD]  = "VHOST_USER_SET_SLAVE_REQ_FD",
  82         [VHOST_USER_IOTLB_MSG]  = "VHOST_USER_IOTLB_MSG",
  83         [VHOST_USER_CRYPTO_CREATE_SESS] = "VHOST_USER_CRYPTO_CREATE_SESS",
  84         [VHOST_USER_CRYPTO_CLOSE_SESS] = "VHOST_USER_CRYPTO_CLOSE_SESS",
  85         [VHOST_USER_POSTCOPY_ADVISE]  = "VHOST_USER_POSTCOPY_ADVISE",
  86         [VHOST_USER_POSTCOPY_LISTEN]  = "VHOST_USER_POSTCOPY_LISTEN",
  87         [VHOST_USER_POSTCOPY_END]  = "VHOST_USER_POSTCOPY_END",
  88         [VHOST_USER_GET_INFLIGHT_FD] = "VHOST_USER_GET_INFLIGHT_FD",
  89         [VHOST_USER_SET_INFLIGHT_FD] = "VHOST_USER_SET_INFLIGHT_FD",
  90         [VHOST_USER_SET_STATUS] = "VHOST_USER_SET_STATUS",
  91 };
  92
  93 static int send_vhost_reply(int sockfd, struct VhostUserMsg *msg);
  94 static int read_vhost_message(int sockfd, struct VhostUserMsg *msg);
  95
  96 static void
  97 close_msg_fds(struct VhostUserMsg *msg)
  98 {
  99         int i;
 100
 101         for (i = 0; i < msg->fd_num; i++)
 102                 close(msg->fds[i]);
 103 }
 104
 105 /*
 106  * Ensure the expected number of FDs is received,
 107  * close all FDs and return an error if this is not the case.
 108  */
 109 static int
 110 validate_msg_fds(struct VhostUserMsg *msg, int expected_fds)
 111 {
 112         if (msg->fd_num == expected_fds)
 113                 return 0;
 114
 115         VHOST_LOG_CONFIG(ERR,
 116                 " Expect %d FDs for request %s, received %d\n",
 117                 expected_fds,
 118                 vhost_message_str[msg->request.master],
 119                 msg->fd_num);
 120
 121         close_msg_fds(msg);
 122
 123         return -1;
 124 }
 125
 126 static uint64_t
 127 get_blk_size(int fd)
 128 {
 129         struct stat stat;
 130         int ret;
 131
 132         ret = fstat(fd, &stat);
 133         return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
 134 }
 135
 136 /*
 137  * Reclaim all the outstanding zmbufs for a virtqueue.
 138  */
 139 static void
 140 drain_zmbuf_list(struct vhost_virtqueue *vq)
 141 {
 142         struct zcopy_mbuf *zmbuf, *next;
 143
 144         for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
 145              zmbuf != NULL; zmbuf = next) {
 146                 next = TAILQ_NEXT(zmbuf, next);
 147
 148                 while (!mbuf_is_consumed(zmbuf->mbuf))
 149                         usleep(1000);
 150
 151                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
 152                 restore_mbuf(zmbuf->mbuf);
 153                 rte_pktmbuf_free(zmbuf->mbuf);
 154                 put_zmbuf(zmbuf);
 155                 vq->nr_zmbuf -= 1;
 156         }
 157 }
 158
 159 static void
 160 free_mem_region(struct virtio_net *dev)
 161 {
 162         uint32_t i;
 163         struct rte_vhost_mem_region *reg;
 164         struct vhost_virtqueue *vq;
 165
 166         if (!dev || !dev->mem)
 167                 return;
 168
 169         if (dev->dequeue_zero_copy) {
 170                 for (i = 0; i < dev->nr_vring; i++) {
 171                         vq = dev->virtqueue[i];
 172                         if (vq)
 173                                 drain_zmbuf_list(vq);
 174                 }
 175         }
 176
 177         for (i = 0; i < dev->mem->nregions; i++) {
 178                 reg = &dev->mem->regions[i];
 179                 if (reg->host_user_addr) {
 180                         munmap(reg->mmap_addr, reg->mmap_size);
 181                         close(reg->fd);
 182                 }
 183         }
 184 }
 185
 186 void
 187 vhost_backend_cleanup(struct virtio_net *dev)
 188 {
 189         if (dev->mem) {
 190                 free_mem_region(dev);
 191                 rte_free(dev->mem);
 192                 dev->mem = NULL;
 193         }
 194
 195         rte_free(dev->guest_pages);
 196         dev->guest_pages = NULL;
 197
 198         if (dev->log_addr) {
 199                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 200                 dev->log_addr = 0;
 201         }
 202
 203         if (dev->inflight_info) {
 204                 if (dev->inflight_info->addr) {
 205                         munmap(dev->inflight_info->addr,
 206                                dev->inflight_info->size);
 207                         dev->inflight_info->addr = NULL;
 208                 }
 209
 210                 if (dev->inflight_info->fd >= 0) {
 211                         close(dev->inflight_info->fd);
 212                         dev->inflight_info->fd = -1;
 213                 }
 214
 215                 free(dev->inflight_info);
 216                 dev->inflight_info = NULL;
 217         }
 218
 219         if (dev->slave_req_fd >= 0) {
 220                 close(dev->slave_req_fd);
 221                 dev->slave_req_fd = -1;
 222         }
 223
 224         if (dev->postcopy_ufd >= 0) {
 225                 close(dev->postcopy_ufd);
 226                 dev->postcopy_ufd = -1;
 227         }
 228
 229         dev->postcopy_listening = 0;
 230 }
 231
 232 static void
 233 vhost_user_notify_queue_state(struct virtio_net *dev, uint16_t index,
 234                               int enable)
 235 {
 236         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
 237
 238         if (vdpa_dev && vdpa_dev->ops->set_vring_state)
 239                 vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
 240
 241         if (dev->notify_ops->vring_state_changed)
 242                 dev->notify_ops->vring_state_changed(dev->vid,
 243                                 index, enable);
 244 }
 245
 246 /*
 247  * This function just returns success at the moment unless
 248  * the device hasn't been initialised.
 249  */
 250 static int
 251 vhost_user_set_owner(struct virtio_net **pdev __rte_unused,
 252                         struct VhostUserMsg *msg,
 253                         int main_fd __rte_unused)
 254 {
 255         if (validate_msg_fds(msg, 0) != 0)
 256                 return RTE_VHOST_MSG_RESULT_ERR;
 257
 258         return RTE_VHOST_MSG_RESULT_OK;
 259 }
 260
 261 static int
 262 vhost_user_reset_owner(struct virtio_net **pdev,
 263                         struct VhostUserMsg *msg,
 264                         int main_fd __rte_unused)
 265 {
 266         struct virtio_net *dev = *pdev;
 267
 268         if (validate_msg_fds(msg, 0) != 0)
 269                 return RTE_VHOST_MSG_RESULT_ERR;
 270
 271         vhost_destroy_device_notify(dev);
 272
 273         cleanup_device(dev, 0);
 274         reset_device(dev);
 275         return RTE_VHOST_MSG_RESULT_OK;
 276 }
 277
 278 /*
 279  * The features that we support are requested.
 280  */
 281 static int
 282 vhost_user_get_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
 283                         int main_fd __rte_unused)
 284 {
 285         struct virtio_net *dev = *pdev;
 286         uint64_t features = 0;
 287
 288         if (validate_msg_fds(msg, 0) != 0)
 289                 return RTE_VHOST_MSG_RESULT_ERR;
 290
 291         rte_vhost_driver_get_features(dev->ifname, &features);
 292
 293         msg->payload.u64 = features;
 294         msg->size = sizeof(msg->payload.u64);
 295         msg->fd_num = 0;
 296
 297         return RTE_VHOST_MSG_RESULT_REPLY;
 298 }
 299
 300 /*
 301  * The queue number that we support are requested.
 302  */
 303 static int
 304 vhost_user_get_queue_num(struct virtio_net **pdev, struct VhostUserMsg *msg,
 305                         int main_fd __rte_unused)
 306 {
 307         struct virtio_net *dev = *pdev;
 308         uint32_t queue_num = 0;
 309
 310         if (validate_msg_fds(msg, 0) != 0)
 311                 return RTE_VHOST_MSG_RESULT_ERR;
 312
 313         rte_vhost_driver_get_queue_num(dev->ifname, &queue_num);
 314
 315         msg->payload.u64 = (uint64_t)queue_num;
 316         msg->size = sizeof(msg->payload.u64);
 317         msg->fd_num = 0;
 318
 319         return RTE_VHOST_MSG_RESULT_REPLY;
 320 }
 321
 322 /*
 323  * We receive the negotiated features supported by us and the virtio device.
 324  */
 325 static int
 326 vhost_user_set_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
 327                         int main_fd __rte_unused)
 328 {
 329         struct virtio_net *dev = *pdev;
 330         uint64_t features = msg->payload.u64;
 331         uint64_t vhost_features = 0;
 332         struct rte_vdpa_device *vdpa_dev;
 333
 334         if (validate_msg_fds(msg, 0) != 0)
 335                 return RTE_VHOST_MSG_RESULT_ERR;
 336
 337         rte_vhost_driver_get_features(dev->ifname, &vhost_features);
 338         if (features & ~vhost_features) {
 339                 VHOST_LOG_CONFIG(ERR,
 340                         "(%d) received invalid negotiated features.\n",
 341                         dev->vid);
 342                 return RTE_VHOST_MSG_RESULT_ERR;
 343         }
 344
 345         if (dev->flags & VIRTIO_DEV_RUNNING) {
 346                 if (dev->features == features)
 347                         return RTE_VHOST_MSG_RESULT_OK;
 348
 349                 /*
 350                  * Error out if master tries to change features while device is
 351                  * in running state. The exception being VHOST_F_LOG_ALL, which
 352                  * is enabled when the live-migration starts.
 353                  */
 354                 if ((dev->features ^ features) & ~(1ULL << VHOST_F_LOG_ALL)) {
 355                         VHOST_LOG_CONFIG(ERR,
 356                                 "(%d) features changed while device is running.\n",
 357                                 dev->vid);
 358                         return RTE_VHOST_MSG_RESULT_ERR;
 359                 }
 360
 361                 if (dev->notify_ops->features_changed)
 362                         dev->notify_ops->features_changed(dev->vid, features);
 363         }
 364
 365         dev->features = features;
 366         if (dev->features &
 367                 ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
 368                 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 369         } else {
 370                 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
 371         }
 372         VHOST_LOG_CONFIG(INFO,
 373                 "negotiated Virtio features: 0x%" PRIx64 "\n", dev->features);
 374         VHOST_LOG_CONFIG(DEBUG,
 375                 "(%d) mergeable RX buffers %s, virtio 1 %s\n",
 376                 dev->vid,
 377                 (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
 378                 (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
 379
 380         if ((dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) &&
 381             !(dev->features & (1ULL << VIRTIO_NET_F_MQ))) {
 382                 /*
 383                  * Remove all but first queue pair if MQ hasn't been
 384                  * negotiated. This is safe because the device is not
 385                  * running at this stage.
 386                  */
 387                 while (dev->nr_vring > 2) {
 388                         struct vhost_virtqueue *vq;
 389
 390                         vq = dev->virtqueue[--dev->nr_vring];
 391                         if (!vq)
 392                                 continue;
 393
 394                         dev->virtqueue[dev->nr_vring] = NULL;
 395                         cleanup_vq(vq, 1);
 396                         cleanup_vq_inflight(dev, vq);
 397                         free_vq(dev, vq);
 398                 }
 399         }
 400
 401         vdpa_dev = dev->vdpa_dev;
 402         if (vdpa_dev)
 403                 vdpa_dev->ops->set_features(dev->vid);
 404
 405         return RTE_VHOST_MSG_RESULT_OK;
 406 }
 407
 408 /*
 409  * The virtio device sends us the size of the descriptor ring.
 410  */
 411 static int
 412 vhost_user_set_vring_num(struct virtio_net **pdev,
 413                         struct VhostUserMsg *msg,
 414                         int main_fd __rte_unused)
 415 {
 416         struct virtio_net *dev = *pdev;
 417         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
 418
 419         if (validate_msg_fds(msg, 0) != 0)
 420                 return RTE_VHOST_MSG_RESULT_ERR;
 421
 422         vq->size = msg->payload.state.num;
 423
 424         /* VIRTIO 1.0, 2.4 Virtqueues says:
 425          *
 426          *   Queue Size value is always a power of 2. The maximum Queue Size
 427          *   value is 32768.
 428          *
 429          * VIRTIO 1.1 2.7 Virtqueues says:
 430          *
 431          *   Packed virtqueues support up to 2^15 entries each.
 432          */
 433         if (!vq_is_packed(dev)) {
 434                 if (vq->size & (vq->size - 1)) {
 435                         VHOST_LOG_CONFIG(ERR,
 436                                 "invalid virtqueue size %u\n", vq->size);
 437                         return RTE_VHOST_MSG_RESULT_ERR;
 438                 }
 439         }
 440
 441         if (vq->size > 32768) {
 442                 VHOST_LOG_CONFIG(ERR,
 443                         "invalid virtqueue size %u\n", vq->size);
 444                 return RTE_VHOST_MSG_RESULT_ERR;
 445         }
 446
 447         if (dev->dequeue_zero_copy) {
 448                 vq->nr_zmbuf = 0;
 449                 vq->last_zmbuf_idx = 0;
 450                 vq->zmbuf_size = vq->size;
 451                 if (vq->zmbufs)
 452                         rte_free(vq->zmbufs);
 453                 vq->zmbufs = rte_zmalloc(NULL, vq->zmbuf_size *
 454                                          sizeof(struct zcopy_mbuf), 0);
 455                 if (vq->zmbufs == NULL) {
 456                         VHOST_LOG_CONFIG(WARNING,
 457                                 "failed to allocate mem for zero copy; "
 458                                 "zero copy is force disabled\n");
 459                         dev->dequeue_zero_copy = 0;
 460                 }
 461                 TAILQ_INIT(&vq->zmbuf_list);
 462         }
 463
 464         if (vq_is_packed(dev)) {
 465                 if (vq->shadow_used_packed)
 466                         rte_free(vq->shadow_used_packed);
 467                 vq->shadow_used_packed = rte_malloc(NULL,
 468                                 vq->size *
 469                                 sizeof(struct vring_used_elem_packed),
 470                                 RTE_CACHE_LINE_SIZE);
 471                 if (!vq->shadow_used_packed) {
 472                         VHOST_LOG_CONFIG(ERR,
 473                                         "failed to allocate memory for shadow used ring.\n");
 474                         return RTE_VHOST_MSG_RESULT_ERR;
 475                 }
 476
 477         } else {
 478                 if (vq->shadow_used_split)
 479                         rte_free(vq->shadow_used_split);
 480
 481                 vq->shadow_used_split = rte_malloc(NULL,
 482                                 vq->size * sizeof(struct vring_used_elem),
 483                                 RTE_CACHE_LINE_SIZE);
 484
 485                 if (!vq->shadow_used_split) {
 486                         VHOST_LOG_CONFIG(ERR,
 487                                         "failed to allocate memory for vq internal data.\n");
 488                         return RTE_VHOST_MSG_RESULT_ERR;
 489                 }
 490         }
 491
 492         if (vq->batch_copy_elems)
 493                 rte_free(vq->batch_copy_elems);
 494         vq->batch_copy_elems = rte_malloc(NULL,
 495                                 vq->size * sizeof(struct batch_copy_elem),
 496                                 RTE_CACHE_LINE_SIZE);
 497         if (!vq->batch_copy_elems) {
 498                 VHOST_LOG_CONFIG(ERR,
 499                         "failed to allocate memory for batching copy.\n");
 500                 return RTE_VHOST_MSG_RESULT_ERR;
 501         }
 502
 503         return RTE_VHOST_MSG_RESULT_OK;
 504 }
 505
 506 /*
 507  * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the
 508  * same numa node as the memory of vring descriptor.
 509  */
 510 #ifdef RTE_LIBRTE_VHOST_NUMA
 511 static struct virtio_net*
 512 numa_realloc(struct virtio_net *dev, int index)
 513 {
 514         int oldnode, newnode;
 515         struct virtio_net *old_dev;
 516         struct vhost_virtqueue *old_vq, *vq;
 517         struct zcopy_mbuf *new_zmbuf;
 518         struct vring_used_elem *new_shadow_used_split;
 519         struct vring_used_elem_packed *new_shadow_used_packed;
 520         struct batch_copy_elem *new_batch_copy_elems;
 521         int ret;
 522
 523         if (dev->flags & VIRTIO_DEV_RUNNING)
 524                 return dev;
 525
 526         old_dev = dev;
 527         vq = old_vq = dev->virtqueue[index];
 528
 529         ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc,
 530                             MPOL_F_NODE | MPOL_F_ADDR);
 531
 532         /* check if we need to reallocate vq */
 533         ret |= get_mempolicy(&oldnode, NULL, 0, old_vq,
 534                              MPOL_F_NODE | MPOL_F_ADDR);
 535         if (ret) {
 536                 VHOST_LOG_CONFIG(ERR,
 537                         "Unable to get vq numa information.\n");
 538                 return dev;
 539         }
 540         if (oldnode != newnode) {
 541                 VHOST_LOG_CONFIG(INFO,
 542                         "reallocate vq from %d to %d node\n", oldnode, newnode);
 543                 vq = rte_malloc_socket(NULL, sizeof(*vq), 0, newnode);
 544                 if (!vq)
 545                         return dev;
 546
 547                 memcpy(vq, old_vq, sizeof(*vq));
 548                 TAILQ_INIT(&vq->zmbuf_list);
 549
 550                 if (dev->dequeue_zero_copy) {
 551                         new_zmbuf = rte_malloc_socket(NULL, vq->zmbuf_size *
 552                                         sizeof(struct zcopy_mbuf), 0, newnode);
 553                         if (new_zmbuf) {
 554                                 rte_free(vq->zmbufs);
 555                                 vq->zmbufs = new_zmbuf;
 556                         }
 557                 }
 558
 559                 if (vq_is_packed(dev)) {
 560                         new_shadow_used_packed = rte_malloc_socket(NULL,
 561                                         vq->size *
 562                                         sizeof(struct vring_used_elem_packed),
 563                                         RTE_CACHE_LINE_SIZE,
 564                                         newnode);
 565                         if (new_shadow_used_packed) {
 566                                 rte_free(vq->shadow_used_packed);
 567                                 vq->shadow_used_packed = new_shadow_used_packed;
 568                         }
 569                 } else {
 570                         new_shadow_used_split = rte_malloc_socket(NULL,
 571                                         vq->size *
 572                                         sizeof(struct vring_used_elem),
 573                                         RTE_CACHE_LINE_SIZE,
 574                                         newnode);
 575                         if (new_shadow_used_split) {
 576                                 rte_free(vq->shadow_used_split);
 577                                 vq->shadow_used_split = new_shadow_used_split;
 578                         }
 579                 }
 580
 581                 new_batch_copy_elems = rte_malloc_socket(NULL,
 582                         vq->size * sizeof(struct batch_copy_elem),
 583                         RTE_CACHE_LINE_SIZE,
 584                         newnode);
 585                 if (new_batch_copy_elems) {
 586                         rte_free(vq->batch_copy_elems);
 587                         vq->batch_copy_elems = new_batch_copy_elems;
 588                 }
 589
 590                 rte_free(old_vq);
 591         }
 592
 593         /* check if we need to reallocate dev */
 594         ret = get_mempolicy(&oldnode, NULL, 0, old_dev,
 595                             MPOL_F_NODE | MPOL_F_ADDR);
 596         if (ret) {
 597                 VHOST_LOG_CONFIG(ERR,
 598                         "Unable to get dev numa information.\n");
 599                 goto out;
 600         }
 601         if (oldnode != newnode) {
 602                 VHOST_LOG_CONFIG(INFO,
 603                         "reallocate dev from %d to %d node\n",
 604                         oldnode, newnode);
 605                 dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode);
 606                 if (!dev) {
 607                         dev = old_dev;
 608                         goto out;
 609                 }
 610
 611                 memcpy(dev, old_dev, sizeof(*dev));
 612                 rte_free(old_dev);
 613         }
 614
 615 out:
 616         dev->virtqueue[index] = vq;
 617         vhost_devices[dev->vid] = dev;
 618
 619         if (old_vq != vq)
 620                 vhost_user_iotlb_init(dev, index);
 621
 622         return dev;
 623 }
 624 #else
 625 static struct virtio_net*
 626 numa_realloc(struct virtio_net *dev, int index __rte_unused)
 627 {
 628         return dev;
 629 }
 630 #endif
 631
 632 /* Converts QEMU virtual address to Vhost virtual address. */
 633 static uint64_t
 634 qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
 635 {
 636         struct rte_vhost_mem_region *r;
 637         uint32_t i;
 638
 639         if (unlikely(!dev || !dev->mem))
 640                 goto out_error;
 641
 642         /* Find the region where the address lives. */
 643         for (i = 0; i < dev->mem->nregions; i++) {
 644                 r = &dev->mem->regions[i];
 645
 646                 if (qva >= r->guest_user_addr &&
 647                     qva <  r->guest_user_addr + r->size) {
 648
 649                         if (unlikely(*len > r->guest_user_addr + r->size - qva))
 650                                 *len = r->guest_user_addr + r->size - qva;
 651
 652                         return qva - r->guest_user_addr +
 653                                r->host_user_addr;
 654                 }
 655         }
 656 out_error:
 657         *len = 0;
 658
 659         return 0;
 660 }
 661
 662
 663 /*
 664  * Converts ring address to Vhost virtual address.
 665  * If IOMMU is enabled, the ring address is a guest IO virtual address,
 666  * else it is a QEMU virtual address.
 667  */
 668 static uint64_t
 669 ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 670                 uint64_t ra, uint64_t *size)
 671 {
 672         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 673                 uint64_t vva;
 674
 675                 vhost_user_iotlb_rd_lock(vq);
 676                 vva = vhost_iova_to_vva(dev, vq, ra,
 677                                         size, VHOST_ACCESS_RW);
 678                 vhost_user_iotlb_rd_unlock(vq);
 679
 680                 return vva;
 681         }
 682
 683         return qva_to_vva(dev, ra, size);
 684 }
 685
 686 static uint64_t
 687 log_addr_to_gpa(struct virtio_net *dev, struct vhost_virtqueue *vq)
 688 {
 689         uint64_t log_gpa;
 690
 691         vhost_user_iotlb_rd_lock(vq);
 692         log_gpa = translate_log_addr(dev, vq, vq->ring_addrs.log_guest_addr);
 693         vhost_user_iotlb_rd_unlock(vq);
 694
 695         return log_gpa;
 696 }
 697
 698 static struct virtio_net *
 699 translate_ring_addresses(struct virtio_net *dev, int vq_index)
 700 {
 701         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 702         struct vhost_vring_addr *addr = &vq->ring_addrs;
 703         uint64_t len, expected_len;
 704
 705         if (addr->flags & (1 << VHOST_VRING_F_LOG)) {
 706                 vq->log_guest_addr =
 707                         log_addr_to_gpa(dev, vq);
 708                 if (vq->log_guest_addr == 0) {
 709                         VHOST_LOG_CONFIG(DEBUG,
 710                                 "(%d) failed to map log_guest_addr.\n",
 711                                 dev->vid);
 712                         return dev;
 713                 }
 714         }
 715
 716         if (vq_is_packed(dev)) {
 717                 len = sizeof(struct vring_packed_desc) * vq->size;
 718                 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 719                         ring_addr_to_vva(dev, vq, addr->desc_user_addr, &len);
 720                 if (vq->desc_packed == NULL ||
 721                                 len != sizeof(struct vring_packed_desc) *
 722                                 vq->size) {
 723                         VHOST_LOG_CONFIG(DEBUG,
 724                                 "(%d) failed to map desc_packed ring.\n",
 725                                 dev->vid);
 726                         return dev;
 727                 }
 728
 729                 dev = numa_realloc(dev, vq_index);
 730                 vq = dev->virtqueue[vq_index];
 731                 addr = &vq->ring_addrs;
 732
 733                 len = sizeof(struct vring_packed_desc_event);
 734                 vq->driver_event = (struct vring_packed_desc_event *)
 735                                         (uintptr_t)ring_addr_to_vva(dev,
 736                                         vq, addr->avail_user_addr, &len);
 737                 if (vq->driver_event == NULL ||
 738                                 len != sizeof(struct vring_packed_desc_event)) {
 739                         VHOST_LOG_CONFIG(DEBUG,
 740                                 "(%d) failed to find driver area address.\n",
 741                                 dev->vid);
 742                         return dev;
 743                 }
 744
 745                 len = sizeof(struct vring_packed_desc_event);
 746                 vq->device_event = (struct vring_packed_desc_event *)
 747                                         (uintptr_t)ring_addr_to_vva(dev,
 748                                         vq, addr->used_user_addr, &len);
 749                 if (vq->device_event == NULL ||
 750                                 len != sizeof(struct vring_packed_desc_event)) {
 751                         VHOST_LOG_CONFIG(DEBUG,
 752                                 "(%d) failed to find device area address.\n",
 753                                 dev->vid);
 754                         return dev;
 755                 }
 756
 757                 vq->access_ok = 1;
 758                 return dev;
 759         }
 760
 761         /* The addresses are converted from QEMU virtual to Vhost virtual. */
 762         if (vq->desc && vq->avail && vq->used)
 763                 return dev;
 764
 765         len = sizeof(struct vring_desc) * vq->size;
 766         vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev,
 767                         vq, addr->desc_user_addr, &len);
 768         if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) {
 769                 VHOST_LOG_CONFIG(DEBUG,
 770                         "(%d) failed to map desc ring.\n",
 771                         dev->vid);
 772                 return dev;
 773         }
 774
 775         dev = numa_realloc(dev, vq_index);
 776         vq = dev->virtqueue[vq_index];
 777         addr = &vq->ring_addrs;
 778
 779         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
 780         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 781                 len += sizeof(uint16_t);
 782         expected_len = len;
 783         vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev,
 784                         vq, addr->avail_user_addr, &len);
 785         if (vq->avail == 0 || len != expected_len) {
 786                 VHOST_LOG_CONFIG(DEBUG,
 787                         "(%d) failed to map avail ring.\n",
 788                         dev->vid);
 789                 return dev;
 790         }
 791
 792         len = sizeof(struct vring_used) +
 793                 sizeof(struct vring_used_elem) * vq->size;
 794         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 795                 len += sizeof(uint16_t);
 796         expected_len = len;
 797         vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev,
 798                         vq, addr->used_user_addr, &len);
 799         if (vq->used == 0 || len != expected_len) {
 800                 VHOST_LOG_CONFIG(DEBUG,
 801                         "(%d) failed to map used ring.\n",
 802                         dev->vid);
 803                 return dev;
 804         }
 805
 806         if (vq->last_used_idx != vq->used->idx) {
 807                 VHOST_LOG_CONFIG(WARNING,
 808                         "last_used_idx (%u) and vq->used->idx (%u) mismatches; "
 809                         "some packets maybe resent for Tx and dropped for Rx\n",
 810                         vq->last_used_idx, vq->used->idx);
 811                 vq->last_used_idx  = vq->used->idx;
 812                 vq->last_avail_idx = vq->used->idx;
 813         }
 814
 815         vq->access_ok = 1;
 816
 817         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address desc: %p\n",
 818                         dev->vid, vq->desc);
 819         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address avail: %p\n",
 820                         dev->vid, vq->avail);
 821         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address used: %p\n",
 822                         dev->vid, vq->used);
 823         VHOST_LOG_CONFIG(DEBUG, "(%d) log_guest_addr: %" PRIx64 "\n",
 824                         dev->vid, vq->log_guest_addr);
 825
 826         return dev;
 827 }
 828
 829 /*
 830  * The virtio device sends us the desc, used and avail ring addresses.
 831  * This function then converts these to our address space.
 832  */
 833 static int
 834 vhost_user_set_vring_addr(struct virtio_net **pdev, struct VhostUserMsg *msg,
 835                         int main_fd __rte_unused)
 836 {
 837         struct virtio_net *dev = *pdev;
 838         struct vhost_virtqueue *vq;
 839         struct vhost_vring_addr *addr = &msg->payload.addr;
 840         bool access_ok;
 841
 842         if (validate_msg_fds(msg, 0) != 0)
 843                 return RTE_VHOST_MSG_RESULT_ERR;
 844
 845         if (dev->mem == NULL)
 846                 return RTE_VHOST_MSG_RESULT_ERR;
 847
 848         /* addr->index refers to the queue index. The txq 1, rxq is 0. */
 849         vq = dev->virtqueue[msg->payload.addr.index];
 850
 851         access_ok = vq->access_ok;
 852
 853         /*
 854          * Rings addresses should not be interpreted as long as the ring is not
 855          * started and enabled
 856          */
 857         memcpy(&vq->ring_addrs, addr, sizeof(*addr));
 858
 859         vring_invalidate(dev, vq);
 860
 861         if ((vq->enabled && (dev->features &
 862                                 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) ||
 863                         access_ok) {
 864                 dev = translate_ring_addresses(dev, msg->payload.addr.index);
 865                 if (!dev)
 866                         return RTE_VHOST_MSG_RESULT_ERR;
 867
 868                 *pdev = dev;
 869         }
 870
 871         return RTE_VHOST_MSG_RESULT_OK;
 872 }
 873
 874 /*
 875  * The virtio device sends us the available ring last used index.
 876  */
 877 static int
 878 vhost_user_set_vring_base(struct virtio_net **pdev,
 879                         struct VhostUserMsg *msg,
 880                         int main_fd __rte_unused)
 881 {
 882         struct virtio_net *dev = *pdev;
 883         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
 884         uint64_t val = msg->payload.state.num;
 885
 886         if (validate_msg_fds(msg, 0) != 0)
 887                 return RTE_VHOST_MSG_RESULT_ERR;
 888
 889         if (vq_is_packed(dev)) {
 890                 /*
 891                  * Bit[0:14]: avail index
 892                  * Bit[15]: avail wrap counter
 893                  */
 894                 vq->last_avail_idx = val & 0x7fff;
 895                 vq->avail_wrap_counter = !!(val & (0x1 << 15));
 896                 /*
 897                  * Set used index to same value as available one, as
 898                  * their values should be the same since ring processing
 899                  * was stopped at get time.
 900                  */
 901                 vq->last_used_idx = vq->last_avail_idx;
 902                 vq->used_wrap_counter = vq->avail_wrap_counter;
 903         } else {
 904                 vq->last_used_idx = msg->payload.state.num;
 905                 vq->last_avail_idx = msg->payload.state.num;
 906         }
 907
 908         return RTE_VHOST_MSG_RESULT_OK;
 909 }
 910
 911 static int
 912 add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
 913                    uint64_t host_phys_addr, uint64_t size)
 914 {
 915         struct guest_page *page, *last_page;
 916         struct guest_page *old_pages;
 917
 918         if (dev->nr_guest_pages == dev->max_guest_pages) {
 919                 dev->max_guest_pages *= 2;
 920                 old_pages = dev->guest_pages;
 921                 dev->guest_pages = rte_realloc(dev->guest_pages,
 922                                         dev->max_guest_pages * sizeof(*page),
 923                                         RTE_CACHE_LINE_SIZE);
 924                 if (dev->guest_pages == NULL) {
 925                         VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
 926                         rte_free(old_pages);
 927                         return -1;
 928                 }
 929         }
 930
 931         if (dev->nr_guest_pages > 0) {
 932                 last_page = &dev->guest_pages[dev->nr_guest_pages - 1];
 933                 /* merge if the two pages are continuous */
 934                 if (host_phys_addr == last_page->host_phys_addr +
 935                                       last_page->size) {
 936                         last_page->size += size;
 937                         return 0;
 938                 }
 939         }
 940
 941         page = &dev->guest_pages[dev->nr_guest_pages++];
 942         page->guest_phys_addr = guest_phys_addr;
 943         page->host_phys_addr  = host_phys_addr;
 944         page->size = size;
 945
 946         return 0;
 947 }
 948
 949 static int
 950 add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
 951                 uint64_t page_size)
 952 {
 953         uint64_t reg_size = reg->size;
 954         uint64_t host_user_addr  = reg->host_user_addr;
 955         uint64_t guest_phys_addr = reg->guest_phys_addr;
 956         uint64_t host_phys_addr;
 957         uint64_t size;
 958
 959         host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr);
 960         size = page_size - (guest_phys_addr & (page_size - 1));
 961         size = RTE_MIN(size, reg_size);
 962
 963         if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size) < 0)
 964                 return -1;
 965
 966         host_user_addr  += size;
 967         guest_phys_addr += size;
 968         reg_size -= size;
 969
 970         while (reg_size > 0) {
 971                 size = RTE_MIN(reg_size, page_size);
 972                 host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)
 973                                                   host_user_addr);
 974                 if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr,
 975                                 size) < 0)
 976                         return -1;
 977
 978                 host_user_addr  += size;
 979                 guest_phys_addr += size;
 980                 reg_size -= size;
 981         }
 982
 983         /* sort guest page array if over binary search threshold */
 984         if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
 985                 qsort((void *)dev->guest_pages, dev->nr_guest_pages,
 986                         sizeof(struct guest_page), guest_page_addrcmp);
 987         }
 988
 989         return 0;
 990 }
 991
 992 #ifdef RTE_LIBRTE_VHOST_DEBUG
 993 /* TODO: enable it only in debug mode? */
 994 static void
 995 dump_guest_pages(struct virtio_net *dev)
 996 {
 997         uint32_t i;
 998         struct guest_page *page;
 999
1000         for (i = 0; i < dev->nr_guest_pages; i++) {
1001                 page = &dev->guest_pages[i];
1002
1003                 VHOST_LOG_CONFIG(INFO,
1004                         "guest physical page region %u\n"
1005                         "\t guest_phys_addr: %" PRIx64 "\n"
1006                         "\t host_phys_addr : %" PRIx64 "\n"
1007                         "\t size           : %" PRIx64 "\n",
1008                         i,
1009                         page->guest_phys_addr,
1010                         page->host_phys_addr,
1011                         page->size);
1012         }
1013 }
1014 #else
1015 #define dump_guest_pages(dev)
1016 #endif
1017
1018 static bool
1019 vhost_memory_changed(struct VhostUserMemory *new,
1020                      struct rte_vhost_memory *old)
1021 {
1022         uint32_t i;
1023
1024         if (new->nregions != old->nregions)
1025                 return true;
1026
1027         for (i = 0; i < new->nregions; ++i) {
1028                 VhostUserMemoryRegion *new_r = &new->regions[i];
1029                 struct rte_vhost_mem_region *old_r = &old->regions[i];
1030
1031                 if (new_r->guest_phys_addr != old_r->guest_phys_addr)
1032                         return true;
1033                 if (new_r->memory_size != old_r->size)
1034                         return true;
1035                 if (new_r->userspace_addr != old_r->guest_user_addr)
1036                         return true;
1037         }
1038
1039         return false;
1040 }
1041
1042 static int
1043 vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *msg,
1044                         int main_fd)
1045 {
1046         struct virtio_net *dev = *pdev;
1047         struct VhostUserMemory *memory = &msg->payload.memory;
1048         struct rte_vhost_mem_region *reg;
1049         void *mmap_addr;
1050         uint64_t mmap_size;
1051         uint64_t mmap_offset;
1052         uint64_t alignment;
1053         uint32_t i;
1054         int populate;
1055         int fd;
1056
1057         if (validate_msg_fds(msg, memory->nregions) != 0)
1058                 return RTE_VHOST_MSG_RESULT_ERR;
1059
1060         if (memory->nregions > VHOST_MEMORY_MAX_NREGIONS) {
1061                 VHOST_LOG_CONFIG(ERR,
1062                         "too many memory regions (%u)\n", memory->nregions);
1063                 return RTE_VHOST_MSG_RESULT_ERR;
1064         }
1065
1066         if (dev->mem && !vhost_memory_changed(memory, dev->mem)) {
1067                 VHOST_LOG_CONFIG(INFO,
1068                         "(%d) memory regions not changed\n", dev->vid);
1069
1070                 close_msg_fds(msg);
1071
1072                 return RTE_VHOST_MSG_RESULT_OK;
1073         }
1074
1075         if (dev->mem) {
1076                 if (dev->flags & VIRTIO_DEV_VDPA_CONFIGURED) {
1077                         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
1078
1079                         if (vdpa_dev && vdpa_dev->ops->dev_close)
1080                                 vdpa_dev->ops->dev_close(dev->vid);
1081                         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
1082                 }
1083                 free_mem_region(dev);
1084                 rte_free(dev->mem);
1085                 dev->mem = NULL;
1086         }
1087
1088         /* Flush IOTLB cache as previous HVAs are now invalid */
1089         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1090                 for (i = 0; i < dev->nr_vring; i++)
1091                         vhost_user_iotlb_flush_all(dev->virtqueue[i]);
1092
1093         dev->nr_guest_pages = 0;
1094         if (dev->guest_pages == NULL) {
1095                 dev->max_guest_pages = 8;
1096                 dev->guest_pages = rte_zmalloc(NULL,
1097                                         dev->max_guest_pages *
1098                                         sizeof(struct guest_page),
1099                                         RTE_CACHE_LINE_SIZE);
1100                 if (dev->guest_pages == NULL) {
1101                         VHOST_LOG_CONFIG(ERR,
1102                                 "(%d) failed to allocate memory "
1103                                 "for dev->guest_pages\n",
1104                                 dev->vid);
1105                         return RTE_VHOST_MSG_RESULT_ERR;
1106                 }
1107         }
1108
1109         dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct rte_vhost_memory) +
1110                 sizeof(struct rte_vhost_mem_region) * memory->nregions, 0);
1111         if (dev->mem == NULL) {
1112                 VHOST_LOG_CONFIG(ERR,
1113                         "(%d) failed to allocate memory for dev->mem\n",
1114                         dev->vid);
1115                 return RTE_VHOST_MSG_RESULT_ERR;
1116         }
1117         dev->mem->nregions = memory->nregions;
1118
1119         for (i = 0; i < memory->nregions; i++) {
1120                 fd  = msg->fds[i];
1121                 reg = &dev->mem->regions[i];
1122
1123                 reg->guest_phys_addr = memory->regions[i].guest_phys_addr;
1124                 reg->guest_user_addr = memory->regions[i].userspace_addr;
1125                 reg->size            = memory->regions[i].memory_size;
1126                 reg->fd              = fd;
1127
1128                 mmap_offset = memory->regions[i].mmap_offset;
1129
1130                 /* Check for memory_size + mmap_offset overflow */
1131                 if (mmap_offset >= -reg->size) {
1132                         VHOST_LOG_CONFIG(ERR,
1133                                 "mmap_offset (%#"PRIx64") and memory_size "
1134                                 "(%#"PRIx64") overflow\n",
1135                                 mmap_offset, reg->size);
1136                         goto err_mmap;
1137                 }
1138
1139                 mmap_size = reg->size + mmap_offset;
1140
1141                 /* mmap() without flag of MAP_ANONYMOUS, should be called
1142                  * with length argument aligned with hugepagesz at older
1143                  * longterm version Linux, like 2.6.32 and 3.2.72, or
1144                  * mmap() will fail with EINVAL.
1145                  *
1146                  * to avoid failure, make sure in caller to keep length
1147                  * aligned.
1148                  */
1149                 alignment = get_blk_size(fd);
1150                 if (alignment == (uint64_t)-1) {
1151                         VHOST_LOG_CONFIG(ERR,
1152                                 "couldn't get hugepage size through fstat\n");
1153                         goto err_mmap;
1154                 }
1155                 mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
1156                 if (mmap_size == 0) {
1157                         /*
1158                          * It could happen if initial mmap_size + alignment
1159                          * overflows the sizeof uint64, which could happen if
1160                          * either mmap_size or alignment value is wrong.
1161                          *
1162                          * mmap() kernel implementation would return an error,
1163                          * but better catch it before and provide useful info
1164                          * in the logs.
1165                          */
1166                         VHOST_LOG_CONFIG(ERR, "mmap size (0x%" PRIx64 ") "
1167                                         "or alignment (0x%" PRIx64 ") is invalid\n",
1168                                         reg->size + mmap_offset, alignment);
1169                         goto err_mmap;
1170                 }
1171
1172                 populate = (dev->dequeue_zero_copy || dev->async_copy) ?
1173                         MAP_POPULATE : 0;
1174                 mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
1175                                  MAP_SHARED | populate, fd, 0);
1176
1177                 if (mmap_addr == MAP_FAILED) {
1178                         VHOST_LOG_CONFIG(ERR,
1179                                 "mmap region %u failed.\n", i);
1180                         goto err_mmap;
1181                 }
1182
1183                 reg->mmap_addr = mmap_addr;
1184                 reg->mmap_size = mmap_size;
1185                 reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr +
1186                                       mmap_offset;
1187
1188                 if (dev->dequeue_zero_copy || dev->async_copy)
1189                         if (add_guest_pages(dev, reg, alignment) < 0) {
1190                                 VHOST_LOG_CONFIG(ERR,
1191                                         "adding guest pages to region %u failed.\n",
1192                                         i);
1193                                 goto err_mmap;
1194                         }
1195
1196                 VHOST_LOG_CONFIG(INFO,
1197                         "guest memory region %u, size: 0x%" PRIx64 "\n"
1198                         "\t guest physical addr: 0x%" PRIx64 "\n"
1199                         "\t guest virtual  addr: 0x%" PRIx64 "\n"
1200                         "\t host  virtual  addr: 0x%" PRIx64 "\n"
1201                         "\t mmap addr : 0x%" PRIx64 "\n"
1202                         "\t mmap size : 0x%" PRIx64 "\n"
1203                         "\t mmap align: 0x%" PRIx64 "\n"
1204                         "\t mmap off  : 0x%" PRIx64 "\n",
1205                         i, reg->size,
1206                         reg->guest_phys_addr,
1207                         reg->guest_user_addr,
1208                         reg->host_user_addr,
1209                         (uint64_t)(uintptr_t)mmap_addr,
1210                         mmap_size,
1211                         alignment,
1212                         mmap_offset);
1213
1214                 if (dev->postcopy_listening) {
1215                         /*
1216                          * We haven't a better way right now than sharing
1217                          * DPDK's virtual address with Qemu, so that Qemu can
1218                          * retrieve the region offset when handling userfaults.
1219                          */
1220                         memory->regions[i].userspace_addr =
1221                                 reg->host_user_addr;
1222                 }
1223         }
1224         if (dev->postcopy_listening) {
1225                 /* Send the addresses back to qemu */
1226                 msg->fd_num = 0;
1227                 send_vhost_reply(main_fd, msg);
1228
1229                 /* Wait for qemu to acknolwedge it's got the addresses
1230                  * we've got to wait before we're allowed to generate faults.
1231                  */
1232                 VhostUserMsg ack_msg;
1233                 if (read_vhost_message(main_fd, &ack_msg) <= 0) {
1234                         VHOST_LOG_CONFIG(ERR,
1235                                 "Failed to read qemu ack on postcopy set-mem-table\n");
1236                         goto err_mmap;
1237                 }
1238
1239                 if (validate_msg_fds(&ack_msg, 0) != 0)
1240                         goto err_mmap;
1241
1242                 if (ack_msg.request.master != VHOST_USER_SET_MEM_TABLE) {
1243                         VHOST_LOG_CONFIG(ERR,
1244                                 "Bad qemu ack on postcopy set-mem-table (%d)\n",
1245                                 ack_msg.request.master);
1246                         goto err_mmap;
1247                 }
1248
1249                 /* Now userfault register and we can use the memory */
1250                 for (i = 0; i < memory->nregions; i++) {
1251 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
1252                         reg = &dev->mem->regions[i];
1253                         struct uffdio_register reg_struct;
1254
1255                         /*
1256                          * Let's register all the mmap'ed area to ensure
1257                          * alignment on page boundary.
1258                          */
1259                         reg_struct.range.start =
1260                                 (uint64_t)(uintptr_t)reg->mmap_addr;
1261                         reg_struct.range.len = reg->mmap_size;
1262                         reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
1263
1264                         if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER,
1265                                                 &reg_struct)) {
1266                                 VHOST_LOG_CONFIG(ERR,
1267                                         "Failed to register ufd for region %d: (ufd = %d) %s\n",
1268                                         i, dev->postcopy_ufd,
1269                                         strerror(errno));
1270                                 goto err_mmap;
1271                         }
1272                         VHOST_LOG_CONFIG(INFO,
1273                                 "\t userfaultfd registered for range : "
1274                                 "%" PRIx64 " - %" PRIx64 "\n",
1275                                 (uint64_t)reg_struct.range.start,
1276                                 (uint64_t)reg_struct.range.start +
1277                                 (uint64_t)reg_struct.range.len - 1);
1278 #else
1279                         goto err_mmap;
1280 #endif
1281                 }
1282         }
1283
1284         for (i = 0; i < dev->nr_vring; i++) {
1285                 struct vhost_virtqueue *vq = dev->virtqueue[i];
1286
1287                 if (vq->desc || vq->avail || vq->used) {
1288                         /*
1289                          * If the memory table got updated, the ring addresses
1290                          * need to be translated again as virtual addresses have
1291                          * changed.
1292                          */
1293                         vring_invalidate(dev, vq);
1294
1295                         dev = translate_ring_addresses(dev, i);
1296                         if (!dev) {
1297                                 dev = *pdev;
1298                                 goto err_mmap;
1299                         }
1300
1301                         *pdev = dev;
1302                 }
1303         }
1304
1305         dump_guest_pages(dev);
1306
1307         return RTE_VHOST_MSG_RESULT_OK;
1308
1309 err_mmap:
1310         free_mem_region(dev);
1311         rte_free(dev->mem);
1312         dev->mem = NULL;
1313         return RTE_VHOST_MSG_RESULT_ERR;
1314 }
1315
1316 static bool
1317 vq_is_ready(struct virtio_net *dev, struct vhost_virtqueue *vq)
1318 {
1319         bool rings_ok;
1320
1321         if (!vq)
1322                 return false;
1323
1324         if (vq_is_packed(dev))
1325                 rings_ok = vq->desc_packed && vq->driver_event &&
1326                         vq->device_event;
1327         else
1328                 rings_ok = vq->desc && vq->avail && vq->used;
1329
1330         return rings_ok &&
1331                vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1332                vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1333                vq->enabled;
1334 }
1335
1336 #define VIRTIO_DEV_NUM_VQS_TO_BE_READY 2u
1337
1338 static int
1339 virtio_is_ready(struct virtio_net *dev)
1340 {
1341         struct vhost_virtqueue *vq;
1342         uint32_t i;
1343
1344         if (dev->flags & VIRTIO_DEV_READY)
1345                 return 1;
1346
1347         if (dev->nr_vring < VIRTIO_DEV_NUM_VQS_TO_BE_READY)
1348                 return 0;
1349
1350         for (i = 0; i < VIRTIO_DEV_NUM_VQS_TO_BE_READY; i++) {
1351                 vq = dev->virtqueue[i];
1352
1353                 if (!vq_is_ready(dev, vq))
1354                         return 0;
1355         }
1356
1357         /* If supported, ensure the frontend is really done with config */
1358         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS))
1359                 if (!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK))
1360                         return 0;
1361
1362         dev->flags |= VIRTIO_DEV_READY;
1363
1364         if (!(dev->flags & VIRTIO_DEV_RUNNING))
1365                 VHOST_LOG_CONFIG(INFO,
1366                         "virtio is now ready for processing.\n");
1367         return 1;
1368 }
1369
1370 static void *
1371 inflight_mem_alloc(const char *name, size_t size, int *fd)
1372 {
1373         void *ptr;
1374         int mfd = -1;
1375         char fname[20] = "/tmp/memfd-XXXXXX";
1376
1377         *fd = -1;
1378 #ifdef MEMFD_SUPPORTED
1379         mfd = memfd_create(name, MFD_CLOEXEC);
1380 #else
1381         RTE_SET_USED(name);
1382 #endif
1383         if (mfd == -1) {
1384                 mfd = mkstemp(fname);
1385                 if (mfd == -1) {
1386                         VHOST_LOG_CONFIG(ERR,
1387                                 "failed to get inflight buffer fd\n");
1388                         return NULL;
1389                 }
1390
1391                 unlink(fname);
1392         }
1393
1394         if (ftruncate(mfd, size) == -1) {
1395                 VHOST_LOG_CONFIG(ERR,
1396                         "failed to alloc inflight buffer\n");
1397                 close(mfd);
1398                 return NULL;
1399         }
1400
1401         ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
1402         if (ptr == MAP_FAILED) {
1403                 VHOST_LOG_CONFIG(ERR,
1404                         "failed to mmap inflight buffer\n");
1405                 close(mfd);
1406                 return NULL;
1407         }
1408
1409         *fd = mfd;
1410         return ptr;
1411 }
1412
1413 static uint32_t
1414 get_pervq_shm_size_split(uint16_t queue_size)
1415 {
1416         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_split) *
1417                                   queue_size + sizeof(uint64_t) +
1418                                   sizeof(uint16_t) * 4, INFLIGHT_ALIGNMENT);
1419 }
1420
1421 static uint32_t
1422 get_pervq_shm_size_packed(uint16_t queue_size)
1423 {
1424         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_packed)
1425                                   * queue_size + sizeof(uint64_t) +
1426                                   sizeof(uint16_t) * 6 + sizeof(uint8_t) * 9,
1427                                   INFLIGHT_ALIGNMENT);
1428 }
1429
1430 static int
1431 vhost_user_get_inflight_fd(struct virtio_net **pdev,
1432                            VhostUserMsg *msg,
1433                            int main_fd __rte_unused)
1434 {
1435         struct rte_vhost_inflight_info_packed *inflight_packed;
1436         uint64_t pervq_inflight_size, mmap_size;
1437         uint16_t num_queues, queue_size;
1438         struct virtio_net *dev = *pdev;
1439         int fd, i, j;
1440         void *addr;
1441
1442         if (msg->size != sizeof(msg->payload.inflight)) {
1443                 VHOST_LOG_CONFIG(ERR,
1444                         "invalid get_inflight_fd message size is %d\n",
1445                         msg->size);
1446                 return RTE_VHOST_MSG_RESULT_ERR;
1447         }
1448
1449         if (dev->inflight_info == NULL) {
1450                 dev->inflight_info = calloc(1,
1451                                             sizeof(struct inflight_mem_info));
1452                 if (!dev->inflight_info) {
1453                         VHOST_LOG_CONFIG(ERR,
1454                                 "failed to alloc dev inflight area\n");
1455                         return RTE_VHOST_MSG_RESULT_ERR;
1456                 }
1457                 dev->inflight_info->fd = -1;
1458         }
1459
1460         num_queues = msg->payload.inflight.num_queues;
1461         queue_size = msg->payload.inflight.queue_size;
1462
1463         VHOST_LOG_CONFIG(INFO, "get_inflight_fd num_queues: %u\n",
1464                 msg->payload.inflight.num_queues);
1465         VHOST_LOG_CONFIG(INFO, "get_inflight_fd queue_size: %u\n",
1466                 msg->payload.inflight.queue_size);
1467
1468         if (vq_is_packed(dev))
1469                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1470         else
1471                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1472
1473         mmap_size = num_queues * pervq_inflight_size;
1474         addr = inflight_mem_alloc("vhost-inflight", mmap_size, &fd);
1475         if (!addr) {
1476                 VHOST_LOG_CONFIG(ERR,
1477                         "failed to alloc vhost inflight area\n");
1478                         msg->payload.inflight.mmap_size = 0;
1479                 return RTE_VHOST_MSG_RESULT_ERR;
1480         }
1481         memset(addr, 0, mmap_size);
1482
1483         if (dev->inflight_info->addr) {
1484                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1485                 dev->inflight_info->addr = NULL;
1486         }
1487
1488         if (dev->inflight_info->fd >= 0) {
1489                 close(dev->inflight_info->fd);
1490                 dev->inflight_info->fd = -1;
1491         }
1492
1493         dev->inflight_info->addr = addr;
1494         dev->inflight_info->size = msg->payload.inflight.mmap_size = mmap_size;
1495         dev->inflight_info->fd = msg->fds[0] = fd;
1496         msg->payload.inflight.mmap_offset = 0;
1497         msg->fd_num = 1;
1498
1499         if (vq_is_packed(dev)) {
1500                 for (i = 0; i < num_queues; i++) {
1501                         inflight_packed =
1502                                 (struct rte_vhost_inflight_info_packed *)addr;
1503                         inflight_packed->used_wrap_counter = 1;
1504                         inflight_packed->old_used_wrap_counter = 1;
1505                         for (j = 0; j < queue_size; j++)
1506                                 inflight_packed->desc[j].next = j + 1;
1507                         addr = (void *)((char *)addr + pervq_inflight_size);
1508                 }
1509         }
1510
1511         VHOST_LOG_CONFIG(INFO,
1512                 "send inflight mmap_size: %"PRIu64"\n",
1513                 msg->payload.inflight.mmap_size);
1514         VHOST_LOG_CONFIG(INFO,
1515                 "send inflight mmap_offset: %"PRIu64"\n",
1516                 msg->payload.inflight.mmap_offset);
1517         VHOST_LOG_CONFIG(INFO,
1518                 "send inflight fd: %d\n", msg->fds[0]);
1519
1520         return RTE_VHOST_MSG_RESULT_REPLY;
1521 }
1522
1523 static int
1524 vhost_user_set_inflight_fd(struct virtio_net **pdev, VhostUserMsg *msg,
1525                            int main_fd __rte_unused)
1526 {
1527         uint64_t mmap_size, mmap_offset;
1528         uint16_t num_queues, queue_size;
1529         struct virtio_net *dev = *pdev;
1530         uint32_t pervq_inflight_size;
1531         struct vhost_virtqueue *vq;
1532         void *addr;
1533         int fd, i;
1534
1535         fd = msg->fds[0];
1536         if (msg->size != sizeof(msg->payload.inflight) || fd < 0) {
1537                 VHOST_LOG_CONFIG(ERR,
1538                         "invalid set_inflight_fd message size is %d,fd is %d\n",
1539                         msg->size, fd);
1540                 return RTE_VHOST_MSG_RESULT_ERR;
1541         }
1542
1543         mmap_size = msg->payload.inflight.mmap_size;
1544         mmap_offset = msg->payload.inflight.mmap_offset;
1545         num_queues = msg->payload.inflight.num_queues;
1546         queue_size = msg->payload.inflight.queue_size;
1547
1548         if (vq_is_packed(dev))
1549                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1550         else
1551                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1552
1553         VHOST_LOG_CONFIG(INFO,
1554                 "set_inflight_fd mmap_size: %"PRIu64"\n", mmap_size);
1555         VHOST_LOG_CONFIG(INFO,
1556                 "set_inflight_fd mmap_offset: %"PRIu64"\n", mmap_offset);
1557         VHOST_LOG_CONFIG(INFO,
1558                 "set_inflight_fd num_queues: %u\n", num_queues);
1559         VHOST_LOG_CONFIG(INFO,
1560                 "set_inflight_fd queue_size: %u\n", queue_size);
1561         VHOST_LOG_CONFIG(INFO,
1562                 "set_inflight_fd fd: %d\n", fd);
1563         VHOST_LOG_CONFIG(INFO,
1564                 "set_inflight_fd pervq_inflight_size: %d\n",
1565                 pervq_inflight_size);
1566
1567         if (!dev->inflight_info) {
1568                 dev->inflight_info = calloc(1,
1569                                             sizeof(struct inflight_mem_info));
1570                 if (dev->inflight_info == NULL) {
1571                         VHOST_LOG_CONFIG(ERR,
1572                                 "failed to alloc dev inflight area\n");
1573                         return RTE_VHOST_MSG_RESULT_ERR;
1574                 }
1575                 dev->inflight_info->fd = -1;
1576         }
1577
1578         if (dev->inflight_info->addr) {
1579                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1580                 dev->inflight_info->addr = NULL;
1581         }
1582
1583         addr = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1584                     fd, mmap_offset);
1585         if (addr == MAP_FAILED) {
1586                 VHOST_LOG_CONFIG(ERR, "failed to mmap share memory.\n");
1587                 return RTE_VHOST_MSG_RESULT_ERR;
1588         }
1589
1590         if (dev->inflight_info->fd >= 0) {
1591                 close(dev->inflight_info->fd);
1592                 dev->inflight_info->fd = -1;
1593         }
1594
1595         dev->inflight_info->fd = fd;
1596         dev->inflight_info->addr = addr;
1597         dev->inflight_info->size = mmap_size;
1598
1599         for (i = 0; i < num_queues; i++) {
1600                 vq = dev->virtqueue[i];
1601                 if (vq_is_packed(dev)) {
1602                         vq->inflight_packed = addr;
1603                         vq->inflight_packed->desc_num = queue_size;
1604                 } else {
1605                         vq->inflight_split = addr;
1606                         vq->inflight_split->desc_num = queue_size;
1607                 }
1608                 addr = (void *)((char *)addr + pervq_inflight_size);
1609         }
1610
1611         return RTE_VHOST_MSG_RESULT_OK;
1612 }
1613
1614 static int
1615 vhost_user_set_vring_call(struct virtio_net **pdev, struct VhostUserMsg *msg,
1616                         int main_fd __rte_unused)
1617 {
1618         struct virtio_net *dev = *pdev;
1619         struct vhost_vring_file file;
1620         struct vhost_virtqueue *vq;
1621         int expected_fds;
1622
1623         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1624         if (validate_msg_fds(msg, expected_fds) != 0)
1625                 return RTE_VHOST_MSG_RESULT_ERR;
1626
1627         file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1628         if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1629                 file.fd = VIRTIO_INVALID_EVENTFD;
1630         else
1631                 file.fd = msg->fds[0];
1632         VHOST_LOG_CONFIG(INFO,
1633                 "vring call idx:%d file:%d\n", file.index, file.fd);
1634
1635         vq = dev->virtqueue[file.index];
1636
1637         if (vq->ready) {
1638                 vhost_user_notify_queue_state(dev, file.index, 0);
1639                 vq->ready = 0;
1640         }
1641
1642         if (vq->callfd >= 0)
1643                 close(vq->callfd);
1644
1645         vq->callfd = file.fd;
1646
1647         return RTE_VHOST_MSG_RESULT_OK;
1648 }
1649
1650 static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
1651                         struct VhostUserMsg *msg,
1652                         int main_fd __rte_unused)
1653 {
1654         int expected_fds;
1655
1656         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1657         if (validate_msg_fds(msg, expected_fds) != 0)
1658                 return RTE_VHOST_MSG_RESULT_ERR;
1659
1660         if (!(msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1661                 close(msg->fds[0]);
1662         VHOST_LOG_CONFIG(INFO, "not implemented\n");
1663
1664         return RTE_VHOST_MSG_RESULT_OK;
1665 }
1666
1667 static int
1668 resubmit_desc_compare(const void *a, const void *b)
1669 {
1670         const struct rte_vhost_resubmit_desc *desc0 = a;
1671         const struct rte_vhost_resubmit_desc *desc1 = b;
1672
1673         if (desc1->counter > desc0->counter)
1674                 return 1;
1675
1676         return -1;
1677 }
1678
1679 static int
1680 vhost_check_queue_inflights_split(struct virtio_net *dev,
1681                                   struct vhost_virtqueue *vq)
1682 {
1683         uint16_t i;
1684         uint16_t resubmit_num = 0, last_io, num;
1685         struct vring_used *used = vq->used;
1686         struct rte_vhost_resubmit_info *resubmit;
1687         struct rte_vhost_inflight_info_split *inflight_split;
1688
1689         if (!(dev->protocol_features &
1690             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1691                 return RTE_VHOST_MSG_RESULT_OK;
1692
1693         /* The frontend may still not support the inflight feature
1694          * although we negotiate the protocol feature.
1695          */
1696         if ((!vq->inflight_split))
1697                 return RTE_VHOST_MSG_RESULT_OK;
1698
1699         if (!vq->inflight_split->version) {
1700                 vq->inflight_split->version = INFLIGHT_VERSION;
1701                 return RTE_VHOST_MSG_RESULT_OK;
1702         }
1703
1704         if (vq->resubmit_inflight)
1705                 return RTE_VHOST_MSG_RESULT_OK;
1706
1707         inflight_split = vq->inflight_split;
1708         vq->global_counter = 0;
1709         last_io = inflight_split->last_inflight_io;
1710
1711         if (inflight_split->used_idx != used->idx) {
1712                 inflight_split->desc[last_io].inflight = 0;
1713                 rte_smp_mb();
1714                 inflight_split->used_idx = used->idx;
1715         }
1716
1717         for (i = 0; i < inflight_split->desc_num; i++) {
1718                 if (inflight_split->desc[i].inflight == 1)
1719                         resubmit_num++;
1720         }
1721
1722         vq->last_avail_idx += resubmit_num;
1723
1724         if (resubmit_num) {
1725                 resubmit  = calloc(1, sizeof(struct rte_vhost_resubmit_info));
1726                 if (!resubmit) {
1727                         VHOST_LOG_CONFIG(ERR,
1728                                 "failed to allocate memory for resubmit info.\n");
1729                         return RTE_VHOST_MSG_RESULT_ERR;
1730                 }
1731
1732                 resubmit->resubmit_list = calloc(resubmit_num,
1733                         sizeof(struct rte_vhost_resubmit_desc));
1734                 if (!resubmit->resubmit_list) {
1735                         VHOST_LOG_CONFIG(ERR,
1736                                 "failed to allocate memory for inflight desc.\n");
1737                         free(resubmit);
1738                         return RTE_VHOST_MSG_RESULT_ERR;
1739                 }
1740
1741                 num = 0;
1742                 for (i = 0; i < vq->inflight_split->desc_num; i++) {
1743                         if (vq->inflight_split->desc[i].inflight == 1) {
1744                                 resubmit->resubmit_list[num].index = i;
1745                                 resubmit->resubmit_list[num].counter =
1746                                         inflight_split->desc[i].counter;
1747                                 num++;
1748                         }
1749                 }
1750                 resubmit->resubmit_num = num;
1751
1752                 if (resubmit->resubmit_num > 1)
1753                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1754                               sizeof(struct rte_vhost_resubmit_desc),
1755                               resubmit_desc_compare);
1756
1757                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1758                 vq->resubmit_inflight = resubmit;
1759         }
1760
1761         return RTE_VHOST_MSG_RESULT_OK;
1762 }
1763
1764 static int
1765 vhost_check_queue_inflights_packed(struct virtio_net *dev,
1766                                    struct vhost_virtqueue *vq)
1767 {
1768         uint16_t i;
1769         uint16_t resubmit_num = 0, old_used_idx, num;
1770         struct rte_vhost_resubmit_info *resubmit;
1771         struct rte_vhost_inflight_info_packed *inflight_packed;
1772
1773         if (!(dev->protocol_features &
1774             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1775                 return RTE_VHOST_MSG_RESULT_OK;
1776
1777         /* The frontend may still not support the inflight feature
1778          * although we negotiate the protocol feature.
1779          */
1780         if ((!vq->inflight_packed))
1781                 return RTE_VHOST_MSG_RESULT_OK;
1782
1783         if (!vq->inflight_packed->version) {
1784                 vq->inflight_packed->version = INFLIGHT_VERSION;
1785                 return RTE_VHOST_MSG_RESULT_OK;
1786         }
1787
1788         if (vq->resubmit_inflight)
1789                 return RTE_VHOST_MSG_RESULT_OK;
1790
1791         inflight_packed = vq->inflight_packed;
1792         vq->global_counter = 0;
1793         old_used_idx = inflight_packed->old_used_idx;
1794
1795         if (inflight_packed->used_idx != old_used_idx) {
1796                 if (inflight_packed->desc[old_used_idx].inflight == 0) {
1797                         inflight_packed->old_used_idx =
1798                                 inflight_packed->used_idx;
1799                         inflight_packed->old_used_wrap_counter =
1800                                 inflight_packed->used_wrap_counter;
1801                         inflight_packed->old_free_head =
1802                                 inflight_packed->free_head;
1803                 } else {
1804                         inflight_packed->used_idx =
1805                                 inflight_packed->old_used_idx;
1806                         inflight_packed->used_wrap_counter =
1807                                 inflight_packed->old_used_wrap_counter;
1808                         inflight_packed->free_head =
1809                                 inflight_packed->old_free_head;
1810                 }
1811         }
1812
1813         for (i = 0; i < inflight_packed->desc_num; i++) {
1814                 if (inflight_packed->desc[i].inflight == 1)
1815                         resubmit_num++;
1816         }
1817
1818         if (resubmit_num) {
1819                 resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
1820                 if (resubmit == NULL) {
1821                         VHOST_LOG_CONFIG(ERR,
1822                                 "failed to allocate memory for resubmit info.\n");
1823                         return RTE_VHOST_MSG_RESULT_ERR;
1824                 }
1825
1826                 resubmit->resubmit_list = calloc(resubmit_num,
1827                         sizeof(struct rte_vhost_resubmit_desc));
1828                 if (resubmit->resubmit_list == NULL) {
1829                         VHOST_LOG_CONFIG(ERR,
1830                                 "failed to allocate memory for resubmit desc.\n");
1831                         free(resubmit);
1832                         return RTE_VHOST_MSG_RESULT_ERR;
1833                 }
1834
1835                 num = 0;
1836                 for (i = 0; i < inflight_packed->desc_num; i++) {
1837                         if (vq->inflight_packed->desc[i].inflight == 1) {
1838                                 resubmit->resubmit_list[num].index = i;
1839                                 resubmit->resubmit_list[num].counter =
1840                                         inflight_packed->desc[i].counter;
1841                                 num++;
1842                         }
1843                 }
1844                 resubmit->resubmit_num = num;
1845
1846                 if (resubmit->resubmit_num > 1)
1847                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1848                               sizeof(struct rte_vhost_resubmit_desc),
1849                               resubmit_desc_compare);
1850
1851                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1852                 vq->resubmit_inflight = resubmit;
1853         }
1854
1855         return RTE_VHOST_MSG_RESULT_OK;
1856 }
1857
1858 static int
1859 vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,
1860                         int main_fd __rte_unused)
1861 {
1862         struct virtio_net *dev = *pdev;
1863         struct vhost_vring_file file;
1864         struct vhost_virtqueue *vq;
1865         int expected_fds;
1866
1867         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1868         if (validate_msg_fds(msg, expected_fds) != 0)
1869                 return RTE_VHOST_MSG_RESULT_ERR;
1870
1871         file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1872         if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1873                 file.fd = VIRTIO_INVALID_EVENTFD;
1874         else
1875                 file.fd = msg->fds[0];
1876         VHOST_LOG_CONFIG(INFO,
1877                 "vring kick idx:%d file:%d\n", file.index, file.fd);
1878
1879         /* Interpret ring addresses only when ring is started. */
1880         dev = translate_ring_addresses(dev, file.index);
1881         if (!dev)
1882                 return RTE_VHOST_MSG_RESULT_ERR;
1883
1884         *pdev = dev;
1885
1886         vq = dev->virtqueue[file.index];
1887
1888         /*
1889          * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated,
1890          * the ring starts already enabled. Otherwise, it is enabled via
1891          * the SET_VRING_ENABLE message.
1892          */
1893         if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) {
1894                 vq->enabled = 1;
1895                 if (dev->notify_ops->vring_state_changed)
1896                         dev->notify_ops->vring_state_changed(
1897                                 dev->vid, file.index, 1);
1898         }
1899
1900         if (vq->ready) {
1901                 vhost_user_notify_queue_state(dev, file.index, 0);
1902                 vq->ready = 0;
1903         }
1904
1905         if (vq->kickfd >= 0)
1906                 close(vq->kickfd);
1907         vq->kickfd = file.fd;
1908
1909         if (vq_is_packed(dev)) {
1910                 if (vhost_check_queue_inflights_packed(dev, vq)) {
1911                         VHOST_LOG_CONFIG(ERR,
1912                                 "failed to inflights for vq: %d\n", file.index);
1913                         return RTE_VHOST_MSG_RESULT_ERR;
1914                 }
1915         } else {
1916                 if (vhost_check_queue_inflights_split(dev, vq)) {
1917                         VHOST_LOG_CONFIG(ERR,
1918                                 "failed to inflights for vq: %d\n", file.index);
1919                         return RTE_VHOST_MSG_RESULT_ERR;
1920                 }
1921         }
1922
1923         return RTE_VHOST_MSG_RESULT_OK;
1924 }
1925
1926 static void
1927 free_zmbufs(struct vhost_virtqueue *vq)
1928 {
1929         drain_zmbuf_list(vq);
1930
1931         rte_free(vq->zmbufs);
1932 }
1933
1934 /*
1935  * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
1936  */
1937 static int
1938 vhost_user_get_vring_base(struct virtio_net **pdev,
1939                         struct VhostUserMsg *msg,
1940                         int main_fd __rte_unused)
1941 {
1942         struct virtio_net *dev = *pdev;
1943         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
1944         uint64_t val;
1945
1946         if (validate_msg_fds(msg, 0) != 0)
1947                 return RTE_VHOST_MSG_RESULT_ERR;
1948
1949         /* We have to stop the queue (virtio) if it is running. */
1950         vhost_destroy_device_notify(dev);
1951
1952         dev->flags &= ~VIRTIO_DEV_READY;
1953         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
1954
1955         /* Here we are safe to get the indexes */
1956         if (vq_is_packed(dev)) {
1957                 /*
1958                  * Bit[0:14]: avail index
1959                  * Bit[15]: avail wrap counter
1960                  */
1961                 val = vq->last_avail_idx & 0x7fff;
1962                 val |= vq->avail_wrap_counter << 15;
1963                 msg->payload.state.num = val;
1964         } else {
1965                 msg->payload.state.num = vq->last_avail_idx;
1966         }
1967
1968         VHOST_LOG_CONFIG(INFO,
1969                 "vring base idx:%d file:%d\n", msg->payload.state.index,
1970                 msg->payload.state.num);
1971         /*
1972          * Based on current qemu vhost-user implementation, this message is
1973          * sent and only sent in vhost_vring_stop.
1974          * TODO: cleanup the vring, it isn't usable since here.
1975          */
1976         if (vq->kickfd >= 0)
1977                 close(vq->kickfd);
1978
1979         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
1980
1981         if (vq->callfd >= 0)
1982                 close(vq->callfd);
1983
1984         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
1985
1986         vq->signalled_used_valid = false;
1987
1988         if (dev->dequeue_zero_copy)
1989                 free_zmbufs(vq);
1990         if (vq_is_packed(dev)) {
1991                 rte_free(vq->shadow_used_packed);
1992                 vq->shadow_used_packed = NULL;
1993         } else {
1994                 rte_free(vq->shadow_used_split);
1995                 vq->shadow_used_split = NULL;
1996                 if (vq->async_pkts_pending)
1997                         rte_free(vq->async_pkts_pending);
1998                 if (vq->async_pending_info)
1999                         rte_free(vq->async_pending_info);
2000                 vq->async_pkts_pending = NULL;
2001                 vq->async_pending_info = NULL;
2002         }
2003
2004         rte_free(vq->batch_copy_elems);
2005         vq->batch_copy_elems = NULL;
2006
2007         msg->size = sizeof(msg->payload.state);
2008         msg->fd_num = 0;
2009
2010         vring_invalidate(dev, vq);
2011
2012         return RTE_VHOST_MSG_RESULT_REPLY;
2013 }
2014
2015 /*
2016  * when virtio queues are ready to work, qemu will send us to
2017  * enable the virtio queue pair.
2018  */
2019 static int
2020 vhost_user_set_vring_enable(struct virtio_net **pdev,
2021                         struct VhostUserMsg *msg,
2022                         int main_fd __rte_unused)
2023 {
2024         struct virtio_net *dev = *pdev;
2025         int enable = (int)msg->payload.state.num;
2026         int index = (int)msg->payload.state.index;
2027
2028         if (validate_msg_fds(msg, 0) != 0)
2029                 return RTE_VHOST_MSG_RESULT_ERR;
2030
2031         VHOST_LOG_CONFIG(INFO,
2032                 "set queue enable: %d to qp idx: %d\n",
2033                 enable, index);
2034
2035         if (!enable && dev->virtqueue[index]->async_registered) {
2036                 if (dev->virtqueue[index]->async_pkts_inflight_n) {
2037                         VHOST_LOG_CONFIG(ERR, "failed to disable vring. "
2038                         "async inflight packets must be completed first\n");
2039                         return RTE_VHOST_MSG_RESULT_ERR;
2040                 }
2041         }
2042
2043         /* On disable, rings have to be stopped being processed. */
2044         if (!enable && dev->dequeue_zero_copy)
2045                 drain_zmbuf_list(dev->virtqueue[index]);
2046
2047         dev->virtqueue[index]->enabled = enable;
2048
2049         return RTE_VHOST_MSG_RESULT_OK;
2050 }
2051
2052 static int
2053 vhost_user_get_protocol_features(struct virtio_net **pdev,
2054                         struct VhostUserMsg *msg,
2055                         int main_fd __rte_unused)
2056 {
2057         struct virtio_net *dev = *pdev;
2058         uint64_t features, protocol_features;
2059
2060         if (validate_msg_fds(msg, 0) != 0)
2061                 return RTE_VHOST_MSG_RESULT_ERR;
2062
2063         rte_vhost_driver_get_features(dev->ifname, &features);
2064         rte_vhost_driver_get_protocol_features(dev->ifname, &protocol_features);
2065
2066         msg->payload.u64 = protocol_features;
2067         msg->size = sizeof(msg->payload.u64);
2068         msg->fd_num = 0;
2069
2070         return RTE_VHOST_MSG_RESULT_REPLY;
2071 }
2072
2073 static int
2074 vhost_user_set_protocol_features(struct virtio_net **pdev,
2075                         struct VhostUserMsg *msg,
2076                         int main_fd __rte_unused)
2077 {
2078         struct virtio_net *dev = *pdev;
2079         uint64_t protocol_features = msg->payload.u64;
2080         uint64_t slave_protocol_features = 0;
2081
2082         if (validate_msg_fds(msg, 0) != 0)
2083                 return RTE_VHOST_MSG_RESULT_ERR;
2084
2085         rte_vhost_driver_get_protocol_features(dev->ifname,
2086                         &slave_protocol_features);
2087         if (protocol_features & ~slave_protocol_features) {
2088                 VHOST_LOG_CONFIG(ERR,
2089                         "(%d) received invalid protocol features.\n",
2090                         dev->vid);
2091                 return RTE_VHOST_MSG_RESULT_ERR;
2092         }
2093
2094         dev->protocol_features = protocol_features;
2095         VHOST_LOG_CONFIG(INFO,
2096                 "negotiated Vhost-user protocol features: 0x%" PRIx64 "\n",
2097                 dev->protocol_features);
2098
2099         return RTE_VHOST_MSG_RESULT_OK;
2100 }
2101
2102 static int
2103 vhost_user_set_log_base(struct virtio_net **pdev, struct VhostUserMsg *msg,
2104                         int main_fd __rte_unused)
2105 {
2106         struct virtio_net *dev = *pdev;
2107         int fd = msg->fds[0];
2108         uint64_t size, off;
2109         void *addr;
2110
2111         if (validate_msg_fds(msg, 1) != 0)
2112                 return RTE_VHOST_MSG_RESULT_ERR;
2113
2114         if (fd < 0) {
2115                 VHOST_LOG_CONFIG(ERR, "invalid log fd: %d\n", fd);
2116                 return RTE_VHOST_MSG_RESULT_ERR;
2117         }
2118
2119         if (msg->size != sizeof(VhostUserLog)) {
2120                 VHOST_LOG_CONFIG(ERR,
2121                         "invalid log base msg size: %"PRId32" != %d\n",
2122                         msg->size, (int)sizeof(VhostUserLog));
2123                 return RTE_VHOST_MSG_RESULT_ERR;
2124         }
2125
2126         size = msg->payload.log.mmap_size;
2127         off  = msg->payload.log.mmap_offset;
2128
2129         /* Check for mmap size and offset overflow. */
2130         if (off >= -size) {
2131                 VHOST_LOG_CONFIG(ERR,
2132                         "log offset %#"PRIx64" and log size %#"PRIx64" overflow\n",
2133                         off, size);
2134                 return RTE_VHOST_MSG_RESULT_ERR;
2135         }
2136
2137         VHOST_LOG_CONFIG(INFO,
2138                 "log mmap size: %"PRId64", offset: %"PRId64"\n",
2139                 size, off);
2140
2141         /*
2142          * mmap from 0 to workaround a hugepage mmap bug: mmap will
2143          * fail when offset is not page size aligned.
2144          */
2145         addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
2146         close(fd);
2147         if (addr == MAP_FAILED) {
2148                 VHOST_LOG_CONFIG(ERR, "mmap log base failed!\n");
2149                 return RTE_VHOST_MSG_RESULT_ERR;
2150         }
2151
2152         /*
2153          * Free previously mapped log memory on occasionally
2154          * multiple VHOST_USER_SET_LOG_BASE.
2155          */
2156         if (dev->log_addr) {
2157                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
2158         }
2159         dev->log_addr = (uint64_t)(uintptr_t)addr;
2160         dev->log_base = dev->log_addr + off;
2161         dev->log_size = size;
2162
2163         /*
2164          * The spec is not clear about it (yet), but QEMU doesn't expect
2165          * any payload in the reply.
2166          */
2167         msg->size = 0;
2168         msg->fd_num = 0;
2169
2170         return RTE_VHOST_MSG_RESULT_REPLY;
2171 }
2172
2173 static int vhost_user_set_log_fd(struct virtio_net **pdev __rte_unused,
2174                         struct VhostUserMsg *msg,
2175                         int main_fd __rte_unused)
2176 {
2177         if (validate_msg_fds(msg, 1) != 0)
2178                 return RTE_VHOST_MSG_RESULT_ERR;
2179
2180         close(msg->fds[0]);
2181         VHOST_LOG_CONFIG(INFO, "not implemented.\n");
2182
2183         return RTE_VHOST_MSG_RESULT_OK;
2184 }
2185
2186 /*
2187  * An rarp packet is constructed and broadcasted to notify switches about
2188  * the new location of the migrated VM, so that packets from outside will
2189  * not be lost after migration.
2190  *
2191  * However, we don't actually "send" a rarp packet here, instead, we set
2192  * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
2193  */
2194 static int
2195 vhost_user_send_rarp(struct virtio_net **pdev, struct VhostUserMsg *msg,
2196                         int main_fd __rte_unused)
2197 {
2198         struct virtio_net *dev = *pdev;
2199         uint8_t *mac = (uint8_t *)&msg->payload.u64;
2200         struct rte_vdpa_device *vdpa_dev;
2201
2202         if (validate_msg_fds(msg, 0) != 0)
2203                 return RTE_VHOST_MSG_RESULT_ERR;
2204
2205         VHOST_LOG_CONFIG(DEBUG,
2206                 ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
2207                 mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
2208         memcpy(dev->mac.addr_bytes, mac, 6);
2209
2210         /*
2211          * Set the flag to inject a RARP broadcast packet at
2212          * rte_vhost_dequeue_burst().
2213          *
2214          * __ATOMIC_RELEASE ordering is for making sure the mac is
2215          * copied before the flag is set.
2216          */
2217         __atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
2218         vdpa_dev = dev->vdpa_dev;
2219         if (vdpa_dev && vdpa_dev->ops->migration_done)
2220                 vdpa_dev->ops->migration_done(dev->vid);
2221
2222         return RTE_VHOST_MSG_RESULT_OK;
2223 }
2224
2225 static int
2226 vhost_user_net_set_mtu(struct virtio_net **pdev, struct VhostUserMsg *msg,
2227                         int main_fd __rte_unused)
2228 {
2229         struct virtio_net *dev = *pdev;
2230
2231         if (validate_msg_fds(msg, 0) != 0)
2232                 return RTE_VHOST_MSG_RESULT_ERR;
2233
2234         if (msg->payload.u64 < VIRTIO_MIN_MTU ||
2235                         msg->payload.u64 > VIRTIO_MAX_MTU) {
2236                 VHOST_LOG_CONFIG(ERR, "Invalid MTU size (%"PRIu64")\n",
2237                                 msg->payload.u64);
2238
2239                 return RTE_VHOST_MSG_RESULT_ERR;
2240         }
2241
2242         dev->mtu = msg->payload.u64;
2243
2244         return RTE_VHOST_MSG_RESULT_OK;
2245 }
2246
2247 static int
2248 vhost_user_set_req_fd(struct virtio_net **pdev, struct VhostUserMsg *msg,
2249                         int main_fd __rte_unused)
2250 {
2251         struct virtio_net *dev = *pdev;
2252         int fd = msg->fds[0];
2253
2254         if (validate_msg_fds(msg, 1) != 0)
2255                 return RTE_VHOST_MSG_RESULT_ERR;
2256
2257         if (fd < 0) {
2258                 VHOST_LOG_CONFIG(ERR,
2259                                 "Invalid file descriptor for slave channel (%d)\n",
2260                                 fd);
2261                 return RTE_VHOST_MSG_RESULT_ERR;
2262         }
2263
2264         if (dev->slave_req_fd >= 0)
2265                 close(dev->slave_req_fd);
2266
2267         dev->slave_req_fd = fd;
2268
2269         return RTE_VHOST_MSG_RESULT_OK;
2270 }
2271
2272 static int
2273 is_vring_iotlb_split(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2274 {
2275         struct vhost_vring_addr *ra;
2276         uint64_t start, end, len;
2277
2278         start = imsg->iova;
2279         end = start + imsg->size;
2280
2281         ra = &vq->ring_addrs;
2282         len = sizeof(struct vring_desc) * vq->size;
2283         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2284                 return 1;
2285
2286         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
2287         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2288                 return 1;
2289
2290         len = sizeof(struct vring_used) +
2291                sizeof(struct vring_used_elem) * vq->size;
2292         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2293                 return 1;
2294
2295         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2296                 len = sizeof(uint64_t);
2297                 if (ra->log_guest_addr < end &&
2298                     (ra->log_guest_addr + len) > start)
2299                         return 1;
2300         }
2301
2302         return 0;
2303 }
2304
2305 static int
2306 is_vring_iotlb_packed(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2307 {
2308         struct vhost_vring_addr *ra;
2309         uint64_t start, end, len;
2310
2311         start = imsg->iova;
2312         end = start + imsg->size;
2313
2314         ra = &vq->ring_addrs;
2315         len = sizeof(struct vring_packed_desc) * vq->size;
2316         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2317                 return 1;
2318
2319         len = sizeof(struct vring_packed_desc_event);
2320         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2321                 return 1;
2322
2323         len = sizeof(struct vring_packed_desc_event);
2324         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2325                 return 1;
2326
2327         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2328                 len = sizeof(uint64_t);
2329                 if (ra->log_guest_addr < end &&
2330                     (ra->log_guest_addr + len) > start)
2331                         return 1;
2332         }
2333
2334         return 0;
2335 }
2336
2337 static int is_vring_iotlb(struct virtio_net *dev,
2338                           struct vhost_virtqueue *vq,
2339                           struct vhost_iotlb_msg *imsg)
2340 {
2341         if (vq_is_packed(dev))
2342                 return is_vring_iotlb_packed(vq, imsg);
2343         else
2344                 return is_vring_iotlb_split(vq, imsg);
2345 }
2346
2347 static int
2348 vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg,
2349                         int main_fd __rte_unused)
2350 {
2351         struct virtio_net *dev = *pdev;
2352         struct vhost_iotlb_msg *imsg = &msg->payload.iotlb;
2353         uint16_t i;
2354         uint64_t vva, len;
2355
2356         if (validate_msg_fds(msg, 0) != 0)
2357                 return RTE_VHOST_MSG_RESULT_ERR;
2358
2359         switch (imsg->type) {
2360         case VHOST_IOTLB_UPDATE:
2361                 len = imsg->size;
2362                 vva = qva_to_vva(dev, imsg->uaddr, &len);
2363                 if (!vva)
2364                         return RTE_VHOST_MSG_RESULT_ERR;
2365
2366                 for (i = 0; i < dev->nr_vring; i++) {
2367                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2368
2369                         vhost_user_iotlb_cache_insert(vq, imsg->iova, vva,
2370                                         len, imsg->perm);
2371
2372                         if (is_vring_iotlb(dev, vq, imsg))
2373                                 *pdev = dev = translate_ring_addresses(dev, i);
2374                 }
2375                 break;
2376         case VHOST_IOTLB_INVALIDATE:
2377                 for (i = 0; i < dev->nr_vring; i++) {
2378                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2379
2380                         vhost_user_iotlb_cache_remove(vq, imsg->iova,
2381                                         imsg->size);
2382
2383                         if (is_vring_iotlb(dev, vq, imsg))
2384                                 vring_invalidate(dev, vq);
2385                 }
2386                 break;
2387         default:
2388                 VHOST_LOG_CONFIG(ERR, "Invalid IOTLB message type (%d)\n",
2389                                 imsg->type);
2390                 return RTE_VHOST_MSG_RESULT_ERR;
2391         }
2392
2393         return RTE_VHOST_MSG_RESULT_OK;
2394 }
2395
2396 static int
2397 vhost_user_set_postcopy_advise(struct virtio_net **pdev,
2398                         struct VhostUserMsg *msg,
2399                         int main_fd __rte_unused)
2400 {
2401         struct virtio_net *dev = *pdev;
2402 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
2403         struct uffdio_api api_struct;
2404
2405         if (validate_msg_fds(msg, 0) != 0)
2406                 return RTE_VHOST_MSG_RESULT_ERR;
2407
2408         dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
2409
2410         if (dev->postcopy_ufd == -1) {
2411                 VHOST_LOG_CONFIG(ERR, "Userfaultfd not available: %s\n",
2412                         strerror(errno));
2413                 return RTE_VHOST_MSG_RESULT_ERR;
2414         }
2415         api_struct.api = UFFD_API;
2416         api_struct.features = 0;
2417         if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
2418                 VHOST_LOG_CONFIG(ERR, "UFFDIO_API ioctl failure: %s\n",
2419                         strerror(errno));
2420                 close(dev->postcopy_ufd);
2421                 dev->postcopy_ufd = -1;
2422                 return RTE_VHOST_MSG_RESULT_ERR;
2423         }
2424         msg->fds[0] = dev->postcopy_ufd;
2425         msg->fd_num = 1;
2426
2427         return RTE_VHOST_MSG_RESULT_REPLY;
2428 #else
2429         dev->postcopy_ufd = -1;
2430         msg->fd_num = 0;
2431
2432         return RTE_VHOST_MSG_RESULT_ERR;
2433 #endif
2434 }
2435
2436 static int
2437 vhost_user_set_postcopy_listen(struct virtio_net **pdev,
2438                         struct VhostUserMsg *msg __rte_unused,
2439                         int main_fd __rte_unused)
2440 {
2441         struct virtio_net *dev = *pdev;
2442
2443         if (validate_msg_fds(msg, 0) != 0)
2444                 return RTE_VHOST_MSG_RESULT_ERR;
2445
2446         if (dev->mem && dev->mem->nregions) {
2447                 VHOST_LOG_CONFIG(ERR,
2448                         "Regions already registered at postcopy-listen\n");
2449                 return RTE_VHOST_MSG_RESULT_ERR;
2450         }
2451         dev->postcopy_listening = 1;
2452
2453         return RTE_VHOST_MSG_RESULT_OK;
2454 }
2455
2456 static int
2457 vhost_user_postcopy_end(struct virtio_net **pdev, struct VhostUserMsg *msg,
2458                         int main_fd __rte_unused)
2459 {
2460         struct virtio_net *dev = *pdev;
2461
2462         if (validate_msg_fds(msg, 0) != 0)
2463                 return RTE_VHOST_MSG_RESULT_ERR;
2464
2465         dev->postcopy_listening = 0;
2466         if (dev->postcopy_ufd >= 0) {
2467                 close(dev->postcopy_ufd);
2468                 dev->postcopy_ufd = -1;
2469         }
2470
2471         msg->payload.u64 = 0;
2472         msg->size = sizeof(msg->payload.u64);
2473         msg->fd_num = 0;
2474
2475         return RTE_VHOST_MSG_RESULT_REPLY;
2476 }
2477
2478 static int
2479 vhost_user_set_status(struct virtio_net **pdev, struct VhostUserMsg *msg,
2480                         int main_fd __rte_unused)
2481 {
2482         struct virtio_net *dev = *pdev;
2483
2484         if (validate_msg_fds(msg, 0) != 0)
2485                 return RTE_VHOST_MSG_RESULT_ERR;
2486
2487         /* As per Virtio specification, the device status is 8bits long */
2488         if (msg->payload.u64 > UINT8_MAX) {
2489                 VHOST_LOG_CONFIG(ERR, "Invalid VHOST_USER_SET_STATUS payload 0x%" PRIx64 "\n",
2490                                 msg->payload.u64);
2491                 return RTE_VHOST_MSG_RESULT_ERR;
2492         }
2493
2494         dev->status = msg->payload.u64;
2495
2496         VHOST_LOG_CONFIG(INFO, "New device status(0x%08x):\n"
2497                         "\t-ACKNOWLEDGE: %u\n"
2498                         "\t-DRIVER: %u\n"
2499                         "\t-FEATURES_OK: %u\n"
2500                         "\t-DRIVER_OK: %u\n"
2501                         "\t-DEVICE_NEED_RESET: %u\n"
2502                         "\t-FAILED: %u\n",
2503                         dev->status,
2504                         !!(dev->status & VIRTIO_DEVICE_STATUS_ACK),
2505                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER),
2506                         !!(dev->status & VIRTIO_DEVICE_STATUS_FEATURES_OK),
2507                         !!(dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK),
2508                         !!(dev->status & VIRTIO_DEVICE_STATUS_DEV_NEED_RESET),
2509                         !!(dev->status & VIRTIO_DEVICE_STATUS_FAILED));
2510
2511         return RTE_VHOST_MSG_RESULT_OK;
2512 }
2513
2514 typedef int (*vhost_message_handler_t)(struct virtio_net **pdev,
2515                                         struct VhostUserMsg *msg,
2516                                         int main_fd);
2517 static vhost_message_handler_t vhost_message_handlers[VHOST_USER_MAX] = {
2518         [VHOST_USER_NONE] = NULL,
2519         [VHOST_USER_GET_FEATURES] = vhost_user_get_features,
2520         [VHOST_USER_SET_FEATURES] = vhost_user_set_features,
2521         [VHOST_USER_SET_OWNER] = vhost_user_set_owner,
2522         [VHOST_USER_RESET_OWNER] = vhost_user_reset_owner,
2523         [VHOST_USER_SET_MEM_TABLE] = vhost_user_set_mem_table,
2524         [VHOST_USER_SET_LOG_BASE] = vhost_user_set_log_base,
2525         [VHOST_USER_SET_LOG_FD] = vhost_user_set_log_fd,
2526         [VHOST_USER_SET_VRING_NUM] = vhost_user_set_vring_num,
2527         [VHOST_USER_SET_VRING_ADDR] = vhost_user_set_vring_addr,
2528         [VHOST_USER_SET_VRING_BASE] = vhost_user_set_vring_base,
2529         [VHOST_USER_GET_VRING_BASE] = vhost_user_get_vring_base,
2530         [VHOST_USER_SET_VRING_KICK] = vhost_user_set_vring_kick,
2531         [VHOST_USER_SET_VRING_CALL] = vhost_user_set_vring_call,
2532         [VHOST_USER_SET_VRING_ERR] = vhost_user_set_vring_err,
2533         [VHOST_USER_GET_PROTOCOL_FEATURES] = vhost_user_get_protocol_features,
2534         [VHOST_USER_SET_PROTOCOL_FEATURES] = vhost_user_set_protocol_features,
2535         [VHOST_USER_GET_QUEUE_NUM] = vhost_user_get_queue_num,
2536         [VHOST_USER_SET_VRING_ENABLE] = vhost_user_set_vring_enable,
2537         [VHOST_USER_SEND_RARP] = vhost_user_send_rarp,
2538         [VHOST_USER_NET_SET_MTU] = vhost_user_net_set_mtu,
2539         [VHOST_USER_SET_SLAVE_REQ_FD] = vhost_user_set_req_fd,
2540         [VHOST_USER_IOTLB_MSG] = vhost_user_iotlb_msg,
2541         [VHOST_USER_POSTCOPY_ADVISE] = vhost_user_set_postcopy_advise,
2542         [VHOST_USER_POSTCOPY_LISTEN] = vhost_user_set_postcopy_listen,
2543         [VHOST_USER_POSTCOPY_END] = vhost_user_postcopy_end,
2544         [VHOST_USER_GET_INFLIGHT_FD] = vhost_user_get_inflight_fd,
2545         [VHOST_USER_SET_INFLIGHT_FD] = vhost_user_set_inflight_fd,
2546         [VHOST_USER_SET_STATUS] = vhost_user_set_status,
2547 };
2548
2549 /* return bytes# of read on success or negative val on failure. */
2550 static int
2551 read_vhost_message(int sockfd, struct VhostUserMsg *msg)
2552 {
2553         int ret;
2554
2555         ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
2556                 msg->fds, VHOST_MEMORY_MAX_NREGIONS, &msg->fd_num);
2557         if (ret <= 0) {
2558                 return ret;
2559         } else if (ret != VHOST_USER_HDR_SIZE) {
2560                 VHOST_LOG_CONFIG(ERR, "Unexpected header size read\n");
2561                 close_msg_fds(msg);
2562                 return -1;
2563         }
2564
2565         if (msg->size) {
2566                 if (msg->size > sizeof(msg->payload)) {
2567                         VHOST_LOG_CONFIG(ERR,
2568                                 "invalid msg size: %d\n", msg->size);
2569                         return -1;
2570                 }
2571                 ret = read(sockfd, &msg->payload, msg->size);
2572                 if (ret <= 0)
2573                         return ret;
2574                 if (ret != (int)msg->size) {
2575                         VHOST_LOG_CONFIG(ERR,
2576                                 "read control message failed\n");
2577                         return -1;
2578                 }
2579         }
2580
2581         return ret;
2582 }
2583
2584 static int
2585 send_vhost_message(int sockfd, struct VhostUserMsg *msg)
2586 {
2587         if (!msg)
2588                 return 0;
2589
2590         return send_fd_message(sockfd, (char *)msg,
2591                 VHOST_USER_HDR_SIZE + msg->size, msg->fds, msg->fd_num);
2592 }
2593
2594 static int
2595 send_vhost_reply(int sockfd, struct VhostUserMsg *msg)
2596 {
2597         if (!msg)
2598                 return 0;
2599
2600         msg->flags &= ~VHOST_USER_VERSION_MASK;
2601         msg->flags &= ~VHOST_USER_NEED_REPLY;
2602         msg->flags |= VHOST_USER_VERSION;
2603         msg->flags |= VHOST_USER_REPLY_MASK;
2604
2605         return send_vhost_message(sockfd, msg);
2606 }
2607
2608 static int
2609 send_vhost_slave_message(struct virtio_net *dev, struct VhostUserMsg *msg)
2610 {
2611         int ret;
2612
2613         if (msg->flags & VHOST_USER_NEED_REPLY)
2614                 rte_spinlock_lock(&dev->slave_req_lock);
2615
2616         ret = send_vhost_message(dev->slave_req_fd, msg);
2617         if (ret < 0 && (msg->flags & VHOST_USER_NEED_REPLY))
2618                 rte_spinlock_unlock(&dev->slave_req_lock);
2619
2620         return ret;
2621 }
2622
2623 /*
2624  * Allocate a queue pair if it hasn't been allocated yet
2625  */
2626 static int
2627 vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev,
2628                         struct VhostUserMsg *msg)
2629 {
2630         uint32_t vring_idx;
2631
2632         switch (msg->request.master) {
2633         case VHOST_USER_SET_VRING_KICK:
2634         case VHOST_USER_SET_VRING_CALL:
2635         case VHOST_USER_SET_VRING_ERR:
2636                 vring_idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
2637                 break;
2638         case VHOST_USER_SET_VRING_NUM:
2639         case VHOST_USER_SET_VRING_BASE:
2640         case VHOST_USER_SET_VRING_ENABLE:
2641                 vring_idx = msg->payload.state.index;
2642                 break;
2643         case VHOST_USER_SET_VRING_ADDR:
2644                 vring_idx = msg->payload.addr.index;
2645                 break;
2646         default:
2647                 return 0;
2648         }
2649
2650         if (vring_idx >= VHOST_MAX_VRING) {
2651                 VHOST_LOG_CONFIG(ERR,
2652                         "invalid vring index: %u\n", vring_idx);
2653                 return -1;
2654         }
2655
2656         if (dev->virtqueue[vring_idx])
2657                 return 0;
2658
2659         return alloc_vring_queue(dev, vring_idx);
2660 }
2661
2662 static void
2663 vhost_user_lock_all_queue_pairs(struct virtio_net *dev)
2664 {
2665         unsigned int i = 0;
2666         unsigned int vq_num = 0;
2667
2668         while (vq_num < dev->nr_vring) {
2669                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2670
2671                 if (vq) {
2672                         rte_spinlock_lock(&vq->access_lock);
2673                         vq_num++;
2674                 }
2675                 i++;
2676         }
2677 }
2678
2679 static void
2680 vhost_user_unlock_all_queue_pairs(struct virtio_net *dev)
2681 {
2682         unsigned int i = 0;
2683         unsigned int vq_num = 0;
2684
2685         while (vq_num < dev->nr_vring) {
2686                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2687
2688                 if (vq) {
2689                         rte_spinlock_unlock(&vq->access_lock);
2690                         vq_num++;
2691                 }
2692                 i++;
2693         }
2694 }
2695
2696 int
2697 vhost_user_msg_handler(int vid, int fd)
2698 {
2699         struct virtio_net *dev;
2700         struct VhostUserMsg msg;
2701         struct rte_vdpa_device *vdpa_dev;
2702         int ret;
2703         int unlock_required = 0;
2704         bool handled;
2705         int request;
2706         uint32_t i;
2707
2708         dev = get_device(vid);
2709         if (dev == NULL)
2710                 return -1;
2711
2712         if (!dev->notify_ops) {
2713                 dev->notify_ops = vhost_driver_callback_get(dev->ifname);
2714                 if (!dev->notify_ops) {
2715                         VHOST_LOG_CONFIG(ERR,
2716                                 "failed to get callback ops for driver %s\n",
2717                                 dev->ifname);
2718                         return -1;
2719                 }
2720         }
2721
2722         ret = read_vhost_message(fd, &msg);
2723         if (ret <= 0) {
2724                 if (ret < 0)
2725                         VHOST_LOG_CONFIG(ERR,
2726                                 "vhost read message failed\n");
2727                 else
2728                         VHOST_LOG_CONFIG(INFO,
2729                                 "vhost peer closed\n");
2730
2731                 return -1;
2732         }
2733
2734         ret = 0;
2735         request = msg.request.master;
2736         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX &&
2737                         vhost_message_str[request]) {
2738                 if (request != VHOST_USER_IOTLB_MSG)
2739                         VHOST_LOG_CONFIG(INFO, "read message %s\n",
2740                                 vhost_message_str[request]);
2741                 else
2742                         VHOST_LOG_CONFIG(DEBUG, "read message %s\n",
2743                                 vhost_message_str[request]);
2744         } else {
2745                 VHOST_LOG_CONFIG(DEBUG, "External request %d\n", request);
2746         }
2747
2748         ret = vhost_user_check_and_alloc_queue_pair(dev, &msg);
2749         if (ret < 0) {
2750                 VHOST_LOG_CONFIG(ERR,
2751                         "failed to alloc queue\n");
2752                 return -1;
2753         }
2754
2755         /*
2756          * Note: we don't lock all queues on VHOST_USER_GET_VRING_BASE
2757          * and VHOST_USER_RESET_OWNER, since it is sent when virtio stops
2758          * and device is destroyed. destroy_device waits for queues to be
2759          * inactive, so it is safe. Otherwise taking the access_lock
2760          * would cause a dead lock.
2761          */
2762         switch (request) {
2763         case VHOST_USER_SET_FEATURES:
2764         case VHOST_USER_SET_PROTOCOL_FEATURES:
2765         case VHOST_USER_SET_OWNER:
2766         case VHOST_USER_SET_MEM_TABLE:
2767         case VHOST_USER_SET_LOG_BASE:
2768         case VHOST_USER_SET_LOG_FD:
2769         case VHOST_USER_SET_VRING_NUM:
2770         case VHOST_USER_SET_VRING_ADDR:
2771         case VHOST_USER_SET_VRING_BASE:
2772         case VHOST_USER_SET_VRING_KICK:
2773         case VHOST_USER_SET_VRING_CALL:
2774         case VHOST_USER_SET_VRING_ERR:
2775         case VHOST_USER_SET_VRING_ENABLE:
2776         case VHOST_USER_SEND_RARP:
2777         case VHOST_USER_NET_SET_MTU:
2778         case VHOST_USER_SET_SLAVE_REQ_FD:
2779                 if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
2780                         vhost_user_lock_all_queue_pairs(dev);
2781                         unlock_required = 1;
2782                 }
2783                 break;
2784         default:
2785                 break;
2786
2787         }
2788
2789         handled = false;
2790         if (dev->extern_ops.pre_msg_handle) {
2791                 ret = (*dev->extern_ops.pre_msg_handle)(dev->vid,
2792                                 (void *)&msg);
2793                 switch (ret) {
2794                 case RTE_VHOST_MSG_RESULT_REPLY:
2795                         send_vhost_reply(fd, &msg);
2796                         /* Fall-through */
2797                 case RTE_VHOST_MSG_RESULT_ERR:
2798                 case RTE_VHOST_MSG_RESULT_OK:
2799                         handled = true;
2800                         goto skip_to_post_handle;
2801                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
2802                 default:
2803                         break;
2804                 }
2805         }
2806
2807         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX) {
2808                 if (!vhost_message_handlers[request])
2809                         goto skip_to_post_handle;
2810                 ret = vhost_message_handlers[request](&dev, &msg, fd);
2811
2812                 switch (ret) {
2813                 case RTE_VHOST_MSG_RESULT_ERR:
2814                         VHOST_LOG_CONFIG(ERR,
2815                                 "Processing %s failed.\n",
2816                                 vhost_message_str[request]);
2817                         handled = true;
2818                         break;
2819                 case RTE_VHOST_MSG_RESULT_OK:
2820                         VHOST_LOG_CONFIG(DEBUG,
2821                                 "Processing %s succeeded.\n",
2822                                 vhost_message_str[request]);
2823                         handled = true;
2824                         break;
2825                 case RTE_VHOST_MSG_RESULT_REPLY:
2826                         VHOST_LOG_CONFIG(DEBUG,
2827                                 "Processing %s succeeded and needs reply.\n",
2828                                 vhost_message_str[request]);
2829                         send_vhost_reply(fd, &msg);
2830                         handled = true;
2831                         break;
2832                 default:
2833                         break;
2834                 }
2835         }
2836
2837 skip_to_post_handle:
2838         if (ret != RTE_VHOST_MSG_RESULT_ERR &&
2839                         dev->extern_ops.post_msg_handle) {
2840                 ret = (*dev->extern_ops.post_msg_handle)(dev->vid,
2841                                 (void *)&msg);
2842                 switch (ret) {
2843                 case RTE_VHOST_MSG_RESULT_REPLY:
2844                         send_vhost_reply(fd, &msg);
2845                         /* Fall-through */
2846                 case RTE_VHOST_MSG_RESULT_ERR:
2847                 case RTE_VHOST_MSG_RESULT_OK:
2848                         handled = true;
2849                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
2850                 default:
2851                         break;
2852                 }
2853         }
2854
2855         if (unlock_required)
2856                 vhost_user_unlock_all_queue_pairs(dev);
2857
2858         /* If message was not handled at this stage, treat it as an error */
2859         if (!handled) {
2860                 VHOST_LOG_CONFIG(ERR,
2861                         "vhost message (req: %d) was not handled.\n", request);
2862                 close_msg_fds(&msg);
2863                 ret = RTE_VHOST_MSG_RESULT_ERR;
2864         }
2865
2866         /*
2867          * If the request required a reply that was already sent,
2868          * this optional reply-ack won't be sent as the
2869          * VHOST_USER_NEED_REPLY was cleared in send_vhost_reply().
2870          */
2871         if (msg.flags & VHOST_USER_NEED_REPLY) {
2872                 msg.payload.u64 = ret == RTE_VHOST_MSG_RESULT_ERR;
2873                 msg.size = sizeof(msg.payload.u64);
2874                 msg.fd_num = 0;
2875                 send_vhost_reply(fd, &msg);
2876         } else if (ret == RTE_VHOST_MSG_RESULT_ERR) {
2877                 VHOST_LOG_CONFIG(ERR,
2878                         "vhost message handling failed.\n");
2879                 return -1;
2880         }
2881
2882         for (i = 0; i < dev->nr_vring; i++) {
2883                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2884                 bool cur_ready = vq_is_ready(dev, vq);
2885
2886                 if (cur_ready != (vq && vq->ready)) {
2887                         vhost_user_notify_queue_state(dev, i, cur_ready);
2888                         vq->ready = cur_ready;
2889                 }
2890         }
2891
2892
2893         if (!virtio_is_ready(dev))
2894                 goto out;
2895
2896         /*
2897          * Virtio is now ready. If not done already, it is time
2898          * to notify the application it can process the rings and
2899          * configure the vDPA device if present.
2900          */
2901
2902         if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
2903                 if (dev->notify_ops->new_device(dev->vid) == 0)
2904                         dev->flags |= VIRTIO_DEV_RUNNING;
2905         }
2906
2907         vdpa_dev = dev->vdpa_dev;
2908         if (!vdpa_dev)
2909                 goto out;
2910
2911         if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
2912                 if (vdpa_dev->ops->dev_conf(dev->vid))
2913                         VHOST_LOG_CONFIG(ERR,
2914                                          "Failed to configure vDPA device\n");
2915                 else
2916                         dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED;
2917         }
2918
2919 out:
2920         return 0;
2921 }
2922
2923 static int process_slave_message_reply(struct virtio_net *dev,
2924                                        const struct VhostUserMsg *msg)
2925 {
2926         struct VhostUserMsg msg_reply;
2927         int ret;
2928
2929         if ((msg->flags & VHOST_USER_NEED_REPLY) == 0)
2930                 return 0;
2931
2932         ret = read_vhost_message(dev->slave_req_fd, &msg_reply);
2933         if (ret <= 0) {
2934                 if (ret < 0)
2935                         VHOST_LOG_CONFIG(ERR,
2936                                 "vhost read slave message reply failed\n");
2937                 else
2938                         VHOST_LOG_CONFIG(INFO,
2939                                 "vhost peer closed\n");
2940                 ret = -1;
2941                 goto out;
2942         }
2943
2944         ret = 0;
2945         if (msg_reply.request.slave != msg->request.slave) {
2946                 VHOST_LOG_CONFIG(ERR,
2947                         "Received unexpected msg type (%u), expected %u\n",
2948                         msg_reply.request.slave, msg->request.slave);
2949                 ret = -1;
2950                 goto out;
2951         }
2952
2953         ret = msg_reply.payload.u64 ? -1 : 0;
2954
2955 out:
2956         rte_spinlock_unlock(&dev->slave_req_lock);
2957         return ret;
2958 }
2959
2960 int
2961 vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
2962 {
2963         int ret;
2964         struct VhostUserMsg msg = {
2965                 .request.slave = VHOST_USER_SLAVE_IOTLB_MSG,
2966                 .flags = VHOST_USER_VERSION,
2967                 .size = sizeof(msg.payload.iotlb),
2968                 .payload.iotlb = {
2969                         .iova = iova,
2970                         .perm = perm,
2971                         .type = VHOST_IOTLB_MISS,
2972                 },
2973         };
2974
2975         ret = send_vhost_message(dev->slave_req_fd, &msg);
2976         if (ret < 0) {
2977                 VHOST_LOG_CONFIG(ERR,
2978                                 "Failed to send IOTLB miss message (%d)\n",
2979                                 ret);
2980                 return ret;
2981         }
2982
2983         return 0;
2984 }
2985
2986 static int
2987 vhost_user_slave_config_change(struct virtio_net *dev, bool need_reply)
2988 {
2989         int ret;
2990         struct VhostUserMsg msg = {
2991                 .request.slave = VHOST_USER_SLAVE_CONFIG_CHANGE_MSG,
2992                 .flags = VHOST_USER_VERSION,
2993                 .size = 0,
2994         };
2995
2996         if (need_reply)
2997                 msg.flags |= VHOST_USER_NEED_REPLY;
2998
2999         ret = send_vhost_slave_message(dev, &msg);
3000         if (ret < 0) {
3001                 VHOST_LOG_CONFIG(ERR,
3002                                 "Failed to send config change (%d)\n",
3003                                 ret);
3004                 return ret;
3005         }
3006
3007         return process_slave_message_reply(dev, &msg);
3008 }
3009
3010 int
3011 rte_vhost_slave_config_change(int vid, bool need_reply)
3012 {
3013         struct virtio_net *dev;
3014
3015         dev = get_device(vid);
3016         if (!dev)
3017                 return -ENODEV;
3018
3019         return vhost_user_slave_config_change(dev, need_reply);
3020 }
3021
3022 static int vhost_user_slave_set_vring_host_notifier(struct virtio_net *dev,
3023                                                     int index, int fd,
3024                                                     uint64_t offset,
3025                                                     uint64_t size)
3026 {
3027         int ret;
3028         struct VhostUserMsg msg = {
3029                 .request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
3030                 .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY,
3031                 .size = sizeof(msg.payload.area),
3032                 .payload.area = {
3033                         .u64 = index & VHOST_USER_VRING_IDX_MASK,
3034                         .size = size,
3035                         .offset = offset,
3036                 },
3037         };
3038
3039         if (fd < 0)
3040                 msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
3041         else {
3042                 msg.fds[0] = fd;
3043                 msg.fd_num = 1;
3044         }
3045
3046         ret = send_vhost_slave_message(dev, &msg);
3047         if (ret < 0) {
3048                 VHOST_LOG_CONFIG(ERR,
3049                         "Failed to set host notifier (%d)\n", ret);
3050                 return ret;
3051         }
3052
3053         return process_slave_message_reply(dev, &msg);
3054 }
3055
3056 int rte_vhost_host_notifier_ctrl(int vid, uint16_t qid, bool enable)
3057 {
3058         struct virtio_net *dev;
3059         struct rte_vdpa_device *vdpa_dev;
3060         int vfio_device_fd, ret = 0;
3061         uint64_t offset, size;
3062         unsigned int i, q_start, q_last;
3063
3064         dev = get_device(vid);
3065         if (!dev)
3066                 return -ENODEV;
3067
3068         vdpa_dev = dev->vdpa_dev;
3069         if (vdpa_dev == NULL)
3070                 return -ENODEV;
3071
3072         if (!(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
3073             !(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) ||
3074             !(dev->protocol_features &
3075                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) ||
3076             !(dev->protocol_features &
3077                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD)) ||
3078             !(dev->protocol_features &
3079                         (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER)))
3080                 return -ENOTSUP;
3081
3082         if (qid == RTE_VHOST_QUEUE_ALL) {
3083                 q_start = 0;
3084                 q_last = dev->nr_vring - 1;
3085         } else {
3086                 if (qid >= dev->nr_vring)
3087                         return -EINVAL;
3088                 q_start = qid;
3089                 q_last = qid;
3090         }
3091
3092         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_vfio_device_fd, -ENOTSUP);
3093         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_notify_area, -ENOTSUP);
3094
3095         vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid);
3096         if (vfio_device_fd < 0)
3097                 return -ENOTSUP;
3098
3099         if (enable) {
3100                 for (i = q_start; i <= q_last; i++) {
3101                         if (vdpa_dev->ops->get_notify_area(vid, i, &offset,
3102                                         &size) < 0) {
3103                                 ret = -ENOTSUP;
3104                                 goto disable;
3105                         }
3106
3107                         if (vhost_user_slave_set_vring_host_notifier(dev, i,
3108                                         vfio_device_fd, offset, size) < 0) {
3109                                 ret = -EFAULT;
3110                                 goto disable;
3111                         }
3112                 }
3113         } else {
3114 disable:
3115                 for (i = q_start; i <= q_last; i++) {
3116                         vhost_user_slave_set_vring_host_notifier(dev, i, -1,
3117                                         0, 0);
3118                 }
3119         }
3120
3121         return ret;
3122 }