lib/librte_vhost/vhost_user.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 /* Security model
   6  * --------------
   7  * The vhost-user protocol connection is an external interface, so it must be
   8  * robust against invalid inputs.
   9  *
  10  * This is important because the vhost-user master is only one step removed
  11  * from the guest.  Malicious guests that have escaped will then launch further
  12  * attacks from the vhost-user master.
  13  *
  14  * Even in deployments where guests are trusted, a bug in the vhost-user master
  15  * can still cause invalid messages to be sent.  Such messages must not
  16  * compromise the stability of the DPDK application by causing crashes, memory
  17  * corruption, or other problematic behavior.
  18  *
  19  * Do not assume received VhostUserMsg fields contain sensible values!
  20  */
  21
  22 #include <stdint.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28 #include <sys/ioctl.h>
  29 #include <sys/mman.h>
  30 #include <sys/types.h>
  31 #include <sys/stat.h>
  32 #include <sys/syscall.h>
  33 #include <assert.h>
  34 #ifdef RTE_LIBRTE_VHOST_NUMA
  35 #include <numaif.h>
  36 #endif
  37 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
  38 #include <linux/userfaultfd.h>
  39 #endif
  40 #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
  41 #include <linux/memfd.h>
  42 #define MEMFD_SUPPORTED
  43 #endif
  44
  45 #include <rte_common.h>
  46 #include <rte_malloc.h>
  47 #include <rte_log.h>
  48
  49 #include "iotlb.h"
  50 #include "vhost.h"
  51 #include "vhost_user.h"
  52
  53 #define VIRTIO_MIN_MTU 68
  54 #define VIRTIO_MAX_MTU 65535
  55
  56 #define INFLIGHT_ALIGNMENT      64
  57 #define INFLIGHT_VERSION        0x1
  58
  59 static const char *vhost_message_str[VHOST_USER_MAX] = {
  60         [VHOST_USER_NONE] = "VHOST_USER_NONE",
  61         [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
  62         [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
  63         [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
  64         [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
  65         [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
  66         [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
  67         [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
  68         [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
  69         [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
  70         [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
  71         [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
  72         [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
  73         [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
  74         [VHOST_USER_SET_VRING_ERR]  = "VHOST_USER_SET_VRING_ERR",
  75         [VHOST_USER_GET_PROTOCOL_FEATURES]  = "VHOST_USER_GET_PROTOCOL_FEATURES",
  76         [VHOST_USER_SET_PROTOCOL_FEATURES]  = "VHOST_USER_SET_PROTOCOL_FEATURES",
  77         [VHOST_USER_GET_QUEUE_NUM]  = "VHOST_USER_GET_QUEUE_NUM",
  78         [VHOST_USER_SET_VRING_ENABLE]  = "VHOST_USER_SET_VRING_ENABLE",
  79         [VHOST_USER_SEND_RARP]  = "VHOST_USER_SEND_RARP",
  80         [VHOST_USER_NET_SET_MTU]  = "VHOST_USER_NET_SET_MTU",
  81         [VHOST_USER_SET_SLAVE_REQ_FD]  = "VHOST_USER_SET_SLAVE_REQ_FD",
  82         [VHOST_USER_IOTLB_MSG]  = "VHOST_USER_IOTLB_MSG",
  83         [VHOST_USER_CRYPTO_CREATE_SESS] = "VHOST_USER_CRYPTO_CREATE_SESS",
  84         [VHOST_USER_CRYPTO_CLOSE_SESS] = "VHOST_USER_CRYPTO_CLOSE_SESS",
  85         [VHOST_USER_POSTCOPY_ADVISE]  = "VHOST_USER_POSTCOPY_ADVISE",
  86         [VHOST_USER_POSTCOPY_LISTEN]  = "VHOST_USER_POSTCOPY_LISTEN",
  87         [VHOST_USER_POSTCOPY_END]  = "VHOST_USER_POSTCOPY_END",
  88         [VHOST_USER_GET_INFLIGHT_FD] = "VHOST_USER_GET_INFLIGHT_FD",
  89         [VHOST_USER_SET_INFLIGHT_FD] = "VHOST_USER_SET_INFLIGHT_FD",
  90 };
  91
  92 static int send_vhost_reply(int sockfd, struct VhostUserMsg *msg);
  93 static int read_vhost_message(int sockfd, struct VhostUserMsg *msg);
  94
  95 static void
  96 close_msg_fds(struct VhostUserMsg *msg)
  97 {
  98         int i;
  99
 100         for (i = 0; i < msg->fd_num; i++)
 101                 close(msg->fds[i]);
 102 }
 103
 104 /*
 105  * Ensure the expected number of FDs is received,
 106  * close all FDs and return an error if this is not the case.
 107  */
 108 static int
 109 validate_msg_fds(struct VhostUserMsg *msg, int expected_fds)
 110 {
 111         if (msg->fd_num == expected_fds)
 112                 return 0;
 113
 114         VHOST_LOG_CONFIG(ERR,
 115                 " Expect %d FDs for request %s, received %d\n",
 116                 expected_fds,
 117                 vhost_message_str[msg->request.master],
 118                 msg->fd_num);
 119
 120         close_msg_fds(msg);
 121
 122         return -1;
 123 }
 124
 125 static uint64_t
 126 get_blk_size(int fd)
 127 {
 128         struct stat stat;
 129         int ret;
 130
 131         ret = fstat(fd, &stat);
 132         return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
 133 }
 134
 135 /*
 136  * Reclaim all the outstanding zmbufs for a virtqueue.
 137  */
 138 static void
 139 drain_zmbuf_list(struct vhost_virtqueue *vq)
 140 {
 141         struct zcopy_mbuf *zmbuf, *next;
 142
 143         for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
 144              zmbuf != NULL; zmbuf = next) {
 145                 next = TAILQ_NEXT(zmbuf, next);
 146
 147                 while (!mbuf_is_consumed(zmbuf->mbuf))
 148                         usleep(1000);
 149
 150                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
 151                 restore_mbuf(zmbuf->mbuf);
 152                 rte_pktmbuf_free(zmbuf->mbuf);
 153                 put_zmbuf(zmbuf);
 154                 vq->nr_zmbuf -= 1;
 155         }
 156 }
 157
 158 static void
 159 free_mem_region(struct virtio_net *dev)
 160 {
 161         uint32_t i;
 162         struct rte_vhost_mem_region *reg;
 163         struct vhost_virtqueue *vq;
 164
 165         if (!dev || !dev->mem)
 166                 return;
 167
 168         if (dev->dequeue_zero_copy) {
 169                 for (i = 0; i < dev->nr_vring; i++) {
 170                         vq = dev->virtqueue[i];
 171                         if (vq)
 172                                 drain_zmbuf_list(vq);
 173                 }
 174         }
 175
 176         for (i = 0; i < dev->mem->nregions; i++) {
 177                 reg = &dev->mem->regions[i];
 178                 if (reg->host_user_addr) {
 179                         munmap(reg->mmap_addr, reg->mmap_size);
 180                         close(reg->fd);
 181                 }
 182         }
 183 }
 184
 185 void
 186 vhost_backend_cleanup(struct virtio_net *dev)
 187 {
 188         if (dev->mem) {
 189                 free_mem_region(dev);
 190                 rte_free(dev->mem);
 191                 dev->mem = NULL;
 192         }
 193
 194         rte_free(dev->guest_pages);
 195         dev->guest_pages = NULL;
 196
 197         if (dev->log_addr) {
 198                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 199                 dev->log_addr = 0;
 200         }
 201
 202         if (dev->inflight_info) {
 203                 if (dev->inflight_info->addr) {
 204                         munmap(dev->inflight_info->addr,
 205                                dev->inflight_info->size);
 206                         dev->inflight_info->addr = NULL;
 207                 }
 208
 209                 if (dev->inflight_info->fd >= 0) {
 210                         close(dev->inflight_info->fd);
 211                         dev->inflight_info->fd = -1;
 212                 }
 213
 214                 free(dev->inflight_info);
 215                 dev->inflight_info = NULL;
 216         }
 217
 218         if (dev->slave_req_fd >= 0) {
 219                 close(dev->slave_req_fd);
 220                 dev->slave_req_fd = -1;
 221         }
 222
 223         if (dev->postcopy_ufd >= 0) {
 224                 close(dev->postcopy_ufd);
 225                 dev->postcopy_ufd = -1;
 226         }
 227
 228         dev->postcopy_listening = 0;
 229 }
 230
 231 static void
 232 vhost_user_notify_queue_state(struct virtio_net *dev, uint16_t index,
 233                               int enable)
 234 {
 235         struct rte_vdpa_device *vdpa_dev = dev->vdpa_dev;
 236
 237         if (vdpa_dev && vdpa_dev->ops->set_vring_state)
 238                 vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
 239
 240         if (dev->notify_ops->vring_state_changed)
 241                 dev->notify_ops->vring_state_changed(dev->vid,
 242                                 index, enable);
 243 }
 244
 245 /*
 246  * This function just returns success at the moment unless
 247  * the device hasn't been initialised.
 248  */
 249 static int
 250 vhost_user_set_owner(struct virtio_net **pdev __rte_unused,
 251                         struct VhostUserMsg *msg,
 252                         int main_fd __rte_unused)
 253 {
 254         if (validate_msg_fds(msg, 0) != 0)
 255                 return RTE_VHOST_MSG_RESULT_ERR;
 256
 257         return RTE_VHOST_MSG_RESULT_OK;
 258 }
 259
 260 static int
 261 vhost_user_reset_owner(struct virtio_net **pdev,
 262                         struct VhostUserMsg *msg,
 263                         int main_fd __rte_unused)
 264 {
 265         struct virtio_net *dev = *pdev;
 266
 267         if (validate_msg_fds(msg, 0) != 0)
 268                 return RTE_VHOST_MSG_RESULT_ERR;
 269
 270         vhost_destroy_device_notify(dev);
 271
 272         cleanup_device(dev, 0);
 273         reset_device(dev);
 274         return RTE_VHOST_MSG_RESULT_OK;
 275 }
 276
 277 /*
 278  * The features that we support are requested.
 279  */
 280 static int
 281 vhost_user_get_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
 282                         int main_fd __rte_unused)
 283 {
 284         struct virtio_net *dev = *pdev;
 285         uint64_t features = 0;
 286
 287         if (validate_msg_fds(msg, 0) != 0)
 288                 return RTE_VHOST_MSG_RESULT_ERR;
 289
 290         rte_vhost_driver_get_features(dev->ifname, &features);
 291
 292         msg->payload.u64 = features;
 293         msg->size = sizeof(msg->payload.u64);
 294         msg->fd_num = 0;
 295
 296         return RTE_VHOST_MSG_RESULT_REPLY;
 297 }
 298
 299 /*
 300  * The queue number that we support are requested.
 301  */
 302 static int
 303 vhost_user_get_queue_num(struct virtio_net **pdev, struct VhostUserMsg *msg,
 304                         int main_fd __rte_unused)
 305 {
 306         struct virtio_net *dev = *pdev;
 307         uint32_t queue_num = 0;
 308
 309         if (validate_msg_fds(msg, 0) != 0)
 310                 return RTE_VHOST_MSG_RESULT_ERR;
 311
 312         rte_vhost_driver_get_queue_num(dev->ifname, &queue_num);
 313
 314         msg->payload.u64 = (uint64_t)queue_num;
 315         msg->size = sizeof(msg->payload.u64);
 316         msg->fd_num = 0;
 317
 318         return RTE_VHOST_MSG_RESULT_REPLY;
 319 }
 320
 321 /*
 322  * We receive the negotiated features supported by us and the virtio device.
 323  */
 324 static int
 325 vhost_user_set_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
 326                         int main_fd __rte_unused)
 327 {
 328         struct virtio_net *dev = *pdev;
 329         uint64_t features = msg->payload.u64;
 330         uint64_t vhost_features = 0;
 331         struct rte_vdpa_device *vdpa_dev;
 332
 333         if (validate_msg_fds(msg, 0) != 0)
 334                 return RTE_VHOST_MSG_RESULT_ERR;
 335
 336         rte_vhost_driver_get_features(dev->ifname, &vhost_features);
 337         if (features & ~vhost_features) {
 338                 VHOST_LOG_CONFIG(ERR,
 339                         "(%d) received invalid negotiated features.\n",
 340                         dev->vid);
 341                 return RTE_VHOST_MSG_RESULT_ERR;
 342         }
 343
 344         if (dev->flags & VIRTIO_DEV_RUNNING) {
 345                 if (dev->features == features)
 346                         return RTE_VHOST_MSG_RESULT_OK;
 347
 348                 /*
 349                  * Error out if master tries to change features while device is
 350                  * in running state. The exception being VHOST_F_LOG_ALL, which
 351                  * is enabled when the live-migration starts.
 352                  */
 353                 if ((dev->features ^ features) & ~(1ULL << VHOST_F_LOG_ALL)) {
 354                         VHOST_LOG_CONFIG(ERR,
 355                                 "(%d) features changed while device is running.\n",
 356                                 dev->vid);
 357                         return RTE_VHOST_MSG_RESULT_ERR;
 358                 }
 359
 360                 if (dev->notify_ops->features_changed)
 361                         dev->notify_ops->features_changed(dev->vid, features);
 362         }
 363
 364         dev->features = features;
 365         if (dev->features &
 366                 ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
 367                 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 368         } else {
 369                 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
 370         }
 371         VHOST_LOG_CONFIG(INFO,
 372                 "negotiated Virtio features: 0x%" PRIx64 "\n", dev->features);
 373         VHOST_LOG_CONFIG(DEBUG,
 374                 "(%d) mergeable RX buffers %s, virtio 1 %s\n",
 375                 dev->vid,
 376                 (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
 377                 (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
 378
 379         if ((dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) &&
 380             !(dev->features & (1ULL << VIRTIO_NET_F_MQ))) {
 381                 /*
 382                  * Remove all but first queue pair if MQ hasn't been
 383                  * negotiated. This is safe because the device is not
 384                  * running at this stage.
 385                  */
 386                 while (dev->nr_vring > 2) {
 387                         struct vhost_virtqueue *vq;
 388
 389                         vq = dev->virtqueue[--dev->nr_vring];
 390                         if (!vq)
 391                                 continue;
 392
 393                         dev->virtqueue[dev->nr_vring] = NULL;
 394                         cleanup_vq(vq, 1);
 395                         cleanup_vq_inflight(dev, vq);
 396                         free_vq(dev, vq);
 397                 }
 398         }
 399
 400         vdpa_dev = dev->vdpa_dev;
 401         if (vdpa_dev && vdpa_dev->ops->set_features)
 402                 vdpa_dev->ops->set_features(dev->vid);
 403
 404         return RTE_VHOST_MSG_RESULT_OK;
 405 }
 406
 407 /*
 408  * The virtio device sends us the size of the descriptor ring.
 409  */
 410 static int
 411 vhost_user_set_vring_num(struct virtio_net **pdev,
 412                         struct VhostUserMsg *msg,
 413                         int main_fd __rte_unused)
 414 {
 415         struct virtio_net *dev = *pdev;
 416         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
 417
 418         if (validate_msg_fds(msg, 0) != 0)
 419                 return RTE_VHOST_MSG_RESULT_ERR;
 420
 421         vq->size = msg->payload.state.num;
 422
 423         /* VIRTIO 1.0, 2.4 Virtqueues says:
 424          *
 425          *   Queue Size value is always a power of 2. The maximum Queue Size
 426          *   value is 32768.
 427          *
 428          * VIRTIO 1.1 2.7 Virtqueues says:
 429          *
 430          *   Packed virtqueues support up to 2^15 entries each.
 431          */
 432         if (!vq_is_packed(dev)) {
 433                 if (vq->size & (vq->size - 1)) {
 434                         VHOST_LOG_CONFIG(ERR,
 435                                 "invalid virtqueue size %u\n", vq->size);
 436                         return RTE_VHOST_MSG_RESULT_ERR;
 437                 }
 438         }
 439
 440         if (vq->size > 32768) {
 441                 VHOST_LOG_CONFIG(ERR,
 442                         "invalid virtqueue size %u\n", vq->size);
 443                 return RTE_VHOST_MSG_RESULT_ERR;
 444         }
 445
 446         if (dev->dequeue_zero_copy) {
 447                 vq->nr_zmbuf = 0;
 448                 vq->last_zmbuf_idx = 0;
 449                 vq->zmbuf_size = vq->size;
 450                 if (vq->zmbufs)
 451                         rte_free(vq->zmbufs);
 452                 vq->zmbufs = rte_zmalloc(NULL, vq->zmbuf_size *
 453                                          sizeof(struct zcopy_mbuf), 0);
 454                 if (vq->zmbufs == NULL) {
 455                         VHOST_LOG_CONFIG(WARNING,
 456                                 "failed to allocate mem for zero copy; "
 457                                 "zero copy is force disabled\n");
 458                         dev->dequeue_zero_copy = 0;
 459                 }
 460                 TAILQ_INIT(&vq->zmbuf_list);
 461         }
 462
 463         if (vq_is_packed(dev)) {
 464                 if (vq->shadow_used_packed)
 465                         rte_free(vq->shadow_used_packed);
 466                 vq->shadow_used_packed = rte_malloc(NULL,
 467                                 vq->size *
 468                                 sizeof(struct vring_used_elem_packed),
 469                                 RTE_CACHE_LINE_SIZE);
 470                 if (!vq->shadow_used_packed) {
 471                         VHOST_LOG_CONFIG(ERR,
 472                                         "failed to allocate memory for shadow used ring.\n");
 473                         return RTE_VHOST_MSG_RESULT_ERR;
 474                 }
 475
 476         } else {
 477                 if (vq->shadow_used_split)
 478                         rte_free(vq->shadow_used_split);
 479                 vq->shadow_used_split = rte_malloc(NULL,
 480                                 vq->size * sizeof(struct vring_used_elem),
 481                                 RTE_CACHE_LINE_SIZE);
 482                 if (!vq->shadow_used_split) {
 483                         VHOST_LOG_CONFIG(ERR,
 484                                         "failed to allocate memory for shadow used ring.\n");
 485                         return RTE_VHOST_MSG_RESULT_ERR;
 486                 }
 487         }
 488
 489         if (vq->batch_copy_elems)
 490                 rte_free(vq->batch_copy_elems);
 491         vq->batch_copy_elems = rte_malloc(NULL,
 492                                 vq->size * sizeof(struct batch_copy_elem),
 493                                 RTE_CACHE_LINE_SIZE);
 494         if (!vq->batch_copy_elems) {
 495                 VHOST_LOG_CONFIG(ERR,
 496                         "failed to allocate memory for batching copy.\n");
 497                 return RTE_VHOST_MSG_RESULT_ERR;
 498         }
 499
 500         return RTE_VHOST_MSG_RESULT_OK;
 501 }
 502
 503 /*
 504  * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the
 505  * same numa node as the memory of vring descriptor.
 506  */
 507 #ifdef RTE_LIBRTE_VHOST_NUMA
 508 static struct virtio_net*
 509 numa_realloc(struct virtio_net *dev, int index)
 510 {
 511         int oldnode, newnode;
 512         struct virtio_net *old_dev;
 513         struct vhost_virtqueue *old_vq, *vq;
 514         struct zcopy_mbuf *new_zmbuf;
 515         struct vring_used_elem *new_shadow_used_split;
 516         struct vring_used_elem_packed *new_shadow_used_packed;
 517         struct batch_copy_elem *new_batch_copy_elems;
 518         int ret;
 519
 520         if (dev->flags & VIRTIO_DEV_RUNNING)
 521                 return dev;
 522
 523         old_dev = dev;
 524         vq = old_vq = dev->virtqueue[index];
 525
 526         ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc,
 527                             MPOL_F_NODE | MPOL_F_ADDR);
 528
 529         /* check if we need to reallocate vq */
 530         ret |= get_mempolicy(&oldnode, NULL, 0, old_vq,
 531                              MPOL_F_NODE | MPOL_F_ADDR);
 532         if (ret) {
 533                 VHOST_LOG_CONFIG(ERR,
 534                         "Unable to get vq numa information.\n");
 535                 return dev;
 536         }
 537         if (oldnode != newnode) {
 538                 VHOST_LOG_CONFIG(INFO,
 539                         "reallocate vq from %d to %d node\n", oldnode, newnode);
 540                 vq = rte_malloc_socket(NULL, sizeof(*vq), 0, newnode);
 541                 if (!vq)
 542                         return dev;
 543
 544                 memcpy(vq, old_vq, sizeof(*vq));
 545                 TAILQ_INIT(&vq->zmbuf_list);
 546
 547                 if (dev->dequeue_zero_copy) {
 548                         new_zmbuf = rte_malloc_socket(NULL, vq->zmbuf_size *
 549                                         sizeof(struct zcopy_mbuf), 0, newnode);
 550                         if (new_zmbuf) {
 551                                 rte_free(vq->zmbufs);
 552                                 vq->zmbufs = new_zmbuf;
 553                         }
 554                 }
 555
 556                 if (vq_is_packed(dev)) {
 557                         new_shadow_used_packed = rte_malloc_socket(NULL,
 558                                         vq->size *
 559                                         sizeof(struct vring_used_elem_packed),
 560                                         RTE_CACHE_LINE_SIZE,
 561                                         newnode);
 562                         if (new_shadow_used_packed) {
 563                                 rte_free(vq->shadow_used_packed);
 564                                 vq->shadow_used_packed = new_shadow_used_packed;
 565                         }
 566                 } else {
 567                         new_shadow_used_split = rte_malloc_socket(NULL,
 568                                         vq->size *
 569                                         sizeof(struct vring_used_elem),
 570                                         RTE_CACHE_LINE_SIZE,
 571                                         newnode);
 572                         if (new_shadow_used_split) {
 573                                 rte_free(vq->shadow_used_split);
 574                                 vq->shadow_used_split = new_shadow_used_split;
 575                         }
 576                 }
 577
 578                 new_batch_copy_elems = rte_malloc_socket(NULL,
 579                         vq->size * sizeof(struct batch_copy_elem),
 580                         RTE_CACHE_LINE_SIZE,
 581                         newnode);
 582                 if (new_batch_copy_elems) {
 583                         rte_free(vq->batch_copy_elems);
 584                         vq->batch_copy_elems = new_batch_copy_elems;
 585                 }
 586
 587                 rte_free(old_vq);
 588         }
 589
 590         /* check if we need to reallocate dev */
 591         ret = get_mempolicy(&oldnode, NULL, 0, old_dev,
 592                             MPOL_F_NODE | MPOL_F_ADDR);
 593         if (ret) {
 594                 VHOST_LOG_CONFIG(ERR,
 595                         "Unable to get dev numa information.\n");
 596                 goto out;
 597         }
 598         if (oldnode != newnode) {
 599                 VHOST_LOG_CONFIG(INFO,
 600                         "reallocate dev from %d to %d node\n",
 601                         oldnode, newnode);
 602                 dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode);
 603                 if (!dev) {
 604                         dev = old_dev;
 605                         goto out;
 606                 }
 607
 608                 memcpy(dev, old_dev, sizeof(*dev));
 609                 rte_free(old_dev);
 610         }
 611
 612 out:
 613         dev->virtqueue[index] = vq;
 614         vhost_devices[dev->vid] = dev;
 615
 616         if (old_vq != vq)
 617                 vhost_user_iotlb_init(dev, index);
 618
 619         return dev;
 620 }
 621 #else
 622 static struct virtio_net*
 623 numa_realloc(struct virtio_net *dev, int index __rte_unused)
 624 {
 625         return dev;
 626 }
 627 #endif
 628
 629 /* Converts QEMU virtual address to Vhost virtual address. */
 630 static uint64_t
 631 qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
 632 {
 633         struct rte_vhost_mem_region *r;
 634         uint32_t i;
 635
 636         if (unlikely(!dev || !dev->mem))
 637                 goto out_error;
 638
 639         /* Find the region where the address lives. */
 640         for (i = 0; i < dev->mem->nregions; i++) {
 641                 r = &dev->mem->regions[i];
 642
 643                 if (qva >= r->guest_user_addr &&
 644                     qva <  r->guest_user_addr + r->size) {
 645
 646                         if (unlikely(*len > r->guest_user_addr + r->size - qva))
 647                                 *len = r->guest_user_addr + r->size - qva;
 648
 649                         return qva - r->guest_user_addr +
 650                                r->host_user_addr;
 651                 }
 652         }
 653 out_error:
 654         *len = 0;
 655
 656         return 0;
 657 }
 658
 659
 660 /*
 661  * Converts ring address to Vhost virtual address.
 662  * If IOMMU is enabled, the ring address is a guest IO virtual address,
 663  * else it is a QEMU virtual address.
 664  */
 665 static uint64_t
 666 ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 667                 uint64_t ra, uint64_t *size)
 668 {
 669         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
 670                 uint64_t vva;
 671
 672                 vhost_user_iotlb_rd_lock(vq);
 673                 vva = vhost_iova_to_vva(dev, vq, ra,
 674                                         size, VHOST_ACCESS_RW);
 675                 vhost_user_iotlb_rd_unlock(vq);
 676
 677                 return vva;
 678         }
 679
 680         return qva_to_vva(dev, ra, size);
 681 }
 682
 683 static uint64_t
 684 log_addr_to_gpa(struct virtio_net *dev, struct vhost_virtqueue *vq)
 685 {
 686         uint64_t log_gpa;
 687
 688         vhost_user_iotlb_rd_lock(vq);
 689         log_gpa = translate_log_addr(dev, vq, vq->ring_addrs.log_guest_addr);
 690         vhost_user_iotlb_rd_unlock(vq);
 691
 692         return log_gpa;
 693 }
 694
 695 static struct virtio_net *
 696 translate_ring_addresses(struct virtio_net *dev, int vq_index)
 697 {
 698         struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
 699         struct vhost_vring_addr *addr = &vq->ring_addrs;
 700         uint64_t len, expected_len;
 701
 702         if (addr->flags & (1 << VHOST_VRING_F_LOG)) {
 703                 vq->log_guest_addr =
 704                         log_addr_to_gpa(dev, vq);
 705                 if (vq->log_guest_addr == 0) {
 706                         VHOST_LOG_CONFIG(DEBUG,
 707                                 "(%d) failed to map log_guest_addr.\n",
 708                                 dev->vid);
 709                         return dev;
 710                 }
 711         }
 712
 713         if (vq_is_packed(dev)) {
 714                 len = sizeof(struct vring_packed_desc) * vq->size;
 715                 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
 716                         ring_addr_to_vva(dev, vq, addr->desc_user_addr, &len);
 717                 if (vq->desc_packed == NULL ||
 718                                 len != sizeof(struct vring_packed_desc) *
 719                                 vq->size) {
 720                         VHOST_LOG_CONFIG(DEBUG,
 721                                 "(%d) failed to map desc_packed ring.\n",
 722                                 dev->vid);
 723                         return dev;
 724                 }
 725
 726                 dev = numa_realloc(dev, vq_index);
 727                 vq = dev->virtqueue[vq_index];
 728                 addr = &vq->ring_addrs;
 729
 730                 len = sizeof(struct vring_packed_desc_event);
 731                 vq->driver_event = (struct vring_packed_desc_event *)
 732                                         (uintptr_t)ring_addr_to_vva(dev,
 733                                         vq, addr->avail_user_addr, &len);
 734                 if (vq->driver_event == NULL ||
 735                                 len != sizeof(struct vring_packed_desc_event)) {
 736                         VHOST_LOG_CONFIG(DEBUG,
 737                                 "(%d) failed to find driver area address.\n",
 738                                 dev->vid);
 739                         return dev;
 740                 }
 741
 742                 len = sizeof(struct vring_packed_desc_event);
 743                 vq->device_event = (struct vring_packed_desc_event *)
 744                                         (uintptr_t)ring_addr_to_vva(dev,
 745                                         vq, addr->used_user_addr, &len);
 746                 if (vq->device_event == NULL ||
 747                                 len != sizeof(struct vring_packed_desc_event)) {
 748                         VHOST_LOG_CONFIG(DEBUG,
 749                                 "(%d) failed to find device area address.\n",
 750                                 dev->vid);
 751                         return dev;
 752                 }
 753
 754                 vq->access_ok = 1;
 755                 return dev;
 756         }
 757
 758         /* The addresses are converted from QEMU virtual to Vhost virtual. */
 759         if (vq->desc && vq->avail && vq->used)
 760                 return dev;
 761
 762         len = sizeof(struct vring_desc) * vq->size;
 763         vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev,
 764                         vq, addr->desc_user_addr, &len);
 765         if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) {
 766                 VHOST_LOG_CONFIG(DEBUG,
 767                         "(%d) failed to map desc ring.\n",
 768                         dev->vid);
 769                 return dev;
 770         }
 771
 772         dev = numa_realloc(dev, vq_index);
 773         vq = dev->virtqueue[vq_index];
 774         addr = &vq->ring_addrs;
 775
 776         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
 777         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 778                 len += sizeof(uint16_t);
 779         expected_len = len;
 780         vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev,
 781                         vq, addr->avail_user_addr, &len);
 782         if (vq->avail == 0 || len != expected_len) {
 783                 VHOST_LOG_CONFIG(DEBUG,
 784                         "(%d) failed to map avail ring.\n",
 785                         dev->vid);
 786                 return dev;
 787         }
 788
 789         len = sizeof(struct vring_used) +
 790                 sizeof(struct vring_used_elem) * vq->size;
 791         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 792                 len += sizeof(uint16_t);
 793         expected_len = len;
 794         vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev,
 795                         vq, addr->used_user_addr, &len);
 796         if (vq->used == 0 || len != expected_len) {
 797                 VHOST_LOG_CONFIG(DEBUG,
 798                         "(%d) failed to map used ring.\n",
 799                         dev->vid);
 800                 return dev;
 801         }
 802
 803         if (vq->last_used_idx != vq->used->idx) {
 804                 VHOST_LOG_CONFIG(WARNING,
 805                         "last_used_idx (%u) and vq->used->idx (%u) mismatches; "
 806                         "some packets maybe resent for Tx and dropped for Rx\n",
 807                         vq->last_used_idx, vq->used->idx);
 808                 vq->last_used_idx  = vq->used->idx;
 809                 vq->last_avail_idx = vq->used->idx;
 810         }
 811
 812         vq->access_ok = 1;
 813
 814         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address desc: %p\n",
 815                         dev->vid, vq->desc);
 816         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address avail: %p\n",
 817                         dev->vid, vq->avail);
 818         VHOST_LOG_CONFIG(DEBUG, "(%d) mapped address used: %p\n",
 819                         dev->vid, vq->used);
 820         VHOST_LOG_CONFIG(DEBUG, "(%d) log_guest_addr: %" PRIx64 "\n",
 821                         dev->vid, vq->log_guest_addr);
 822
 823         return dev;
 824 }
 825
 826 /*
 827  * The virtio device sends us the desc, used and avail ring addresses.
 828  * This function then converts these to our address space.
 829  */
 830 static int
 831 vhost_user_set_vring_addr(struct virtio_net **pdev, struct VhostUserMsg *msg,
 832                         int main_fd __rte_unused)
 833 {
 834         struct virtio_net *dev = *pdev;
 835         struct vhost_virtqueue *vq;
 836         struct vhost_vring_addr *addr = &msg->payload.addr;
 837         bool access_ok;
 838
 839         if (validate_msg_fds(msg, 0) != 0)
 840                 return RTE_VHOST_MSG_RESULT_ERR;
 841
 842         if (dev->mem == NULL)
 843                 return RTE_VHOST_MSG_RESULT_ERR;
 844
 845         /* addr->index refers to the queue index. The txq 1, rxq is 0. */
 846         vq = dev->virtqueue[msg->payload.addr.index];
 847
 848         access_ok = vq->access_ok;
 849
 850         /*
 851          * Rings addresses should not be interpreted as long as the ring is not
 852          * started and enabled
 853          */
 854         memcpy(&vq->ring_addrs, addr, sizeof(*addr));
 855
 856         vring_invalidate(dev, vq);
 857
 858         if ((vq->enabled && (dev->features &
 859                                 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) ||
 860                         access_ok) {
 861                 dev = translate_ring_addresses(dev, msg->payload.addr.index);
 862                 if (!dev)
 863                         return RTE_VHOST_MSG_RESULT_ERR;
 864
 865                 *pdev = dev;
 866         }
 867
 868         return RTE_VHOST_MSG_RESULT_OK;
 869 }
 870
 871 /*
 872  * The virtio device sends us the available ring last used index.
 873  */
 874 static int
 875 vhost_user_set_vring_base(struct virtio_net **pdev,
 876                         struct VhostUserMsg *msg,
 877                         int main_fd __rte_unused)
 878 {
 879         struct virtio_net *dev = *pdev;
 880         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
 881         uint64_t val = msg->payload.state.num;
 882
 883         if (validate_msg_fds(msg, 0) != 0)
 884                 return RTE_VHOST_MSG_RESULT_ERR;
 885
 886         if (vq_is_packed(dev)) {
 887                 /*
 888                  * Bit[0:14]: avail index
 889                  * Bit[15]: avail wrap counter
 890                  */
 891                 vq->last_avail_idx = val & 0x7fff;
 892                 vq->avail_wrap_counter = !!(val & (0x1 << 15));
 893                 /*
 894                  * Set used index to same value as available one, as
 895                  * their values should be the same since ring processing
 896                  * was stopped at get time.
 897                  */
 898                 vq->last_used_idx = vq->last_avail_idx;
 899                 vq->used_wrap_counter = vq->avail_wrap_counter;
 900         } else {
 901                 vq->last_used_idx = msg->payload.state.num;
 902                 vq->last_avail_idx = msg->payload.state.num;
 903         }
 904
 905         return RTE_VHOST_MSG_RESULT_OK;
 906 }
 907
 908 static int
 909 add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
 910                    uint64_t host_phys_addr, uint64_t size)
 911 {
 912         struct guest_page *page, *last_page;
 913         struct guest_page *old_pages;
 914
 915         if (dev->nr_guest_pages == dev->max_guest_pages) {
 916                 dev->max_guest_pages *= 2;
 917                 old_pages = dev->guest_pages;
 918                 dev->guest_pages = rte_realloc(dev->guest_pages,
 919                                         dev->max_guest_pages * sizeof(*page),
 920                                         RTE_CACHE_LINE_SIZE);
 921                 if (dev->guest_pages == NULL) {
 922                         VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
 923                         rte_free(old_pages);
 924                         return -1;
 925                 }
 926         }
 927
 928         if (dev->nr_guest_pages > 0) {
 929                 last_page = &dev->guest_pages[dev->nr_guest_pages - 1];
 930                 /* merge if the two pages are continuous */
 931                 if (host_phys_addr == last_page->host_phys_addr +
 932                                       last_page->size) {
 933                         last_page->size += size;
 934                         return 0;
 935                 }
 936         }
 937
 938         page = &dev->guest_pages[dev->nr_guest_pages++];
 939         page->guest_phys_addr = guest_phys_addr;
 940         page->host_phys_addr  = host_phys_addr;
 941         page->size = size;
 942
 943         return 0;
 944 }
 945
 946 static int
 947 add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
 948                 uint64_t page_size)
 949 {
 950         uint64_t reg_size = reg->size;
 951         uint64_t host_user_addr  = reg->host_user_addr;
 952         uint64_t guest_phys_addr = reg->guest_phys_addr;
 953         uint64_t host_phys_addr;
 954         uint64_t size;
 955
 956         host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr);
 957         size = page_size - (guest_phys_addr & (page_size - 1));
 958         size = RTE_MIN(size, reg_size);
 959
 960         if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size) < 0)
 961                 return -1;
 962
 963         host_user_addr  += size;
 964         guest_phys_addr += size;
 965         reg_size -= size;
 966
 967         while (reg_size > 0) {
 968                 size = RTE_MIN(reg_size, page_size);
 969                 host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)
 970                                                   host_user_addr);
 971                 if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr,
 972                                 size) < 0)
 973                         return -1;
 974
 975                 host_user_addr  += size;
 976                 guest_phys_addr += size;
 977                 reg_size -= size;
 978         }
 979
 980         /* sort guest page array if over binary search threshold */
 981         if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
 982                 qsort((void *)dev->guest_pages, dev->nr_guest_pages,
 983                         sizeof(struct guest_page), guest_page_addrcmp);
 984         }
 985
 986         return 0;
 987 }
 988
 989 #ifdef RTE_LIBRTE_VHOST_DEBUG
 990 /* TODO: enable it only in debug mode? */
 991 static void
 992 dump_guest_pages(struct virtio_net *dev)
 993 {
 994         uint32_t i;
 995         struct guest_page *page;
 996
 997         for (i = 0; i < dev->nr_guest_pages; i++) {
 998                 page = &dev->guest_pages[i];
 999
1000                 VHOST_LOG_CONFIG(INFO,
1001                         "guest physical page region %u\n"
1002                         "\t guest_phys_addr: %" PRIx64 "\n"
1003                         "\t host_phys_addr : %" PRIx64 "\n"
1004                         "\t size           : %" PRIx64 "\n",
1005                         i,
1006                         page->guest_phys_addr,
1007                         page->host_phys_addr,
1008                         page->size);
1009         }
1010 }
1011 #else
1012 #define dump_guest_pages(dev)
1013 #endif
1014
1015 static bool
1016 vhost_memory_changed(struct VhostUserMemory *new,
1017                      struct rte_vhost_memory *old)
1018 {
1019         uint32_t i;
1020
1021         if (new->nregions != old->nregions)
1022                 return true;
1023
1024         for (i = 0; i < new->nregions; ++i) {
1025                 VhostUserMemoryRegion *new_r = &new->regions[i];
1026                 struct rte_vhost_mem_region *old_r = &old->regions[i];
1027
1028                 if (new_r->guest_phys_addr != old_r->guest_phys_addr)
1029                         return true;
1030                 if (new_r->memory_size != old_r->size)
1031                         return true;
1032                 if (new_r->userspace_addr != old_r->guest_user_addr)
1033                         return true;
1034         }
1035
1036         return false;
1037 }
1038
1039 static int
1040 vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *msg,
1041                         int main_fd)
1042 {
1043         struct virtio_net *dev = *pdev;
1044         struct VhostUserMemory *memory = &msg->payload.memory;
1045         struct rte_vhost_mem_region *reg;
1046         void *mmap_addr;
1047         uint64_t mmap_size;
1048         uint64_t mmap_offset;
1049         uint64_t alignment;
1050         uint32_t i;
1051         int populate;
1052         int fd;
1053
1054         if (validate_msg_fds(msg, memory->nregions) != 0)
1055                 return RTE_VHOST_MSG_RESULT_ERR;
1056
1057         if (memory->nregions > VHOST_MEMORY_MAX_NREGIONS) {
1058                 VHOST_LOG_CONFIG(ERR,
1059                         "too many memory regions (%u)\n", memory->nregions);
1060                 return RTE_VHOST_MSG_RESULT_ERR;
1061         }
1062
1063         if (dev->mem && !vhost_memory_changed(memory, dev->mem)) {
1064                 VHOST_LOG_CONFIG(INFO,
1065                         "(%d) memory regions not changed\n", dev->vid);
1066
1067                 close_msg_fds(msg);
1068
1069                 return RTE_VHOST_MSG_RESULT_OK;
1070         }
1071
1072         if (dev->mem) {
1073                 free_mem_region(dev);
1074                 rte_free(dev->mem);
1075                 dev->mem = NULL;
1076         }
1077
1078         /* Flush IOTLB cache as previous HVAs are now invalid */
1079         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1080                 for (i = 0; i < dev->nr_vring; i++)
1081                         vhost_user_iotlb_flush_all(dev->virtqueue[i]);
1082
1083         dev->nr_guest_pages = 0;
1084         if (dev->guest_pages == NULL) {
1085                 dev->max_guest_pages = 8;
1086                 dev->guest_pages = rte_zmalloc(NULL,
1087                                         dev->max_guest_pages *
1088                                         sizeof(struct guest_page),
1089                                         RTE_CACHE_LINE_SIZE);
1090                 if (dev->guest_pages == NULL) {
1091                         VHOST_LOG_CONFIG(ERR,
1092                                 "(%d) failed to allocate memory "
1093                                 "for dev->guest_pages\n",
1094                                 dev->vid);
1095                         return RTE_VHOST_MSG_RESULT_ERR;
1096                 }
1097         }
1098
1099         dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct rte_vhost_memory) +
1100                 sizeof(struct rte_vhost_mem_region) * memory->nregions, 0);
1101         if (dev->mem == NULL) {
1102                 VHOST_LOG_CONFIG(ERR,
1103                         "(%d) failed to allocate memory for dev->mem\n",
1104                         dev->vid);
1105                 return RTE_VHOST_MSG_RESULT_ERR;
1106         }
1107         dev->mem->nregions = memory->nregions;
1108
1109         for (i = 0; i < memory->nregions; i++) {
1110                 fd  = msg->fds[i];
1111                 reg = &dev->mem->regions[i];
1112
1113                 reg->guest_phys_addr = memory->regions[i].guest_phys_addr;
1114                 reg->guest_user_addr = memory->regions[i].userspace_addr;
1115                 reg->size            = memory->regions[i].memory_size;
1116                 reg->fd              = fd;
1117
1118                 mmap_offset = memory->regions[i].mmap_offset;
1119
1120                 /* Check for memory_size + mmap_offset overflow */
1121                 if (mmap_offset >= -reg->size) {
1122                         VHOST_LOG_CONFIG(ERR,
1123                                 "mmap_offset (%#"PRIx64") and memory_size "
1124                                 "(%#"PRIx64") overflow\n",
1125                                 mmap_offset, reg->size);
1126                         goto err_mmap;
1127                 }
1128
1129                 mmap_size = reg->size + mmap_offset;
1130
1131                 /* mmap() without flag of MAP_ANONYMOUS, should be called
1132                  * with length argument aligned with hugepagesz at older
1133                  * longterm version Linux, like 2.6.32 and 3.2.72, or
1134                  * mmap() will fail with EINVAL.
1135                  *
1136                  * to avoid failure, make sure in caller to keep length
1137                  * aligned.
1138                  */
1139                 alignment = get_blk_size(fd);
1140                 if (alignment == (uint64_t)-1) {
1141                         VHOST_LOG_CONFIG(ERR,
1142                                 "couldn't get hugepage size through fstat\n");
1143                         goto err_mmap;
1144                 }
1145                 mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
1146                 if (mmap_size == 0) {
1147                         /*
1148                          * It could happen if initial mmap_size + alignment
1149                          * overflows the sizeof uint64, which could happen if
1150                          * either mmap_size or alignment value is wrong.
1151                          *
1152                          * mmap() kernel implementation would return an error,
1153                          * but better catch it before and provide useful info
1154                          * in the logs.
1155                          */
1156                         VHOST_LOG_CONFIG(ERR, "mmap size (0x%" PRIx64 ") "
1157                                         "or alignment (0x%" PRIx64 ") is invalid\n",
1158                                         reg->size + mmap_offset, alignment);
1159                         goto err_mmap;
1160                 }
1161
1162                 populate = (dev->dequeue_zero_copy) ? MAP_POPULATE : 0;
1163                 mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
1164                                  MAP_SHARED | populate, fd, 0);
1165
1166                 if (mmap_addr == MAP_FAILED) {
1167                         VHOST_LOG_CONFIG(ERR,
1168                                 "mmap region %u failed.\n", i);
1169                         goto err_mmap;
1170                 }
1171
1172                 reg->mmap_addr = mmap_addr;
1173                 reg->mmap_size = mmap_size;
1174                 reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr +
1175                                       mmap_offset;
1176
1177                 if (dev->dequeue_zero_copy)
1178                         if (add_guest_pages(dev, reg, alignment) < 0) {
1179                                 VHOST_LOG_CONFIG(ERR,
1180                                         "adding guest pages to region %u failed.\n",
1181                                         i);
1182                                 goto err_mmap;
1183                         }
1184
1185                 VHOST_LOG_CONFIG(INFO,
1186                         "guest memory region %u, size: 0x%" PRIx64 "\n"
1187                         "\t guest physical addr: 0x%" PRIx64 "\n"
1188                         "\t guest virtual  addr: 0x%" PRIx64 "\n"
1189                         "\t host  virtual  addr: 0x%" PRIx64 "\n"
1190                         "\t mmap addr : 0x%" PRIx64 "\n"
1191                         "\t mmap size : 0x%" PRIx64 "\n"
1192                         "\t mmap align: 0x%" PRIx64 "\n"
1193                         "\t mmap off  : 0x%" PRIx64 "\n",
1194                         i, reg->size,
1195                         reg->guest_phys_addr,
1196                         reg->guest_user_addr,
1197                         reg->host_user_addr,
1198                         (uint64_t)(uintptr_t)mmap_addr,
1199                         mmap_size,
1200                         alignment,
1201                         mmap_offset);
1202
1203                 if (dev->postcopy_listening) {
1204                         /*
1205                          * We haven't a better way right now than sharing
1206                          * DPDK's virtual address with Qemu, so that Qemu can
1207                          * retrieve the region offset when handling userfaults.
1208                          */
1209                         memory->regions[i].userspace_addr =
1210                                 reg->host_user_addr;
1211                 }
1212         }
1213         if (dev->postcopy_listening) {
1214                 /* Send the addresses back to qemu */
1215                 msg->fd_num = 0;
1216                 send_vhost_reply(main_fd, msg);
1217
1218                 /* Wait for qemu to acknolwedge it's got the addresses
1219                  * we've got to wait before we're allowed to generate faults.
1220                  */
1221                 VhostUserMsg ack_msg;
1222                 if (read_vhost_message(main_fd, &ack_msg) <= 0) {
1223                         VHOST_LOG_CONFIG(ERR,
1224                                 "Failed to read qemu ack on postcopy set-mem-table\n");
1225                         goto err_mmap;
1226                 }
1227
1228                 if (validate_msg_fds(&ack_msg, 0) != 0)
1229                         goto err_mmap;
1230
1231                 if (ack_msg.request.master != VHOST_USER_SET_MEM_TABLE) {
1232                         VHOST_LOG_CONFIG(ERR,
1233                                 "Bad qemu ack on postcopy set-mem-table (%d)\n",
1234                                 ack_msg.request.master);
1235                         goto err_mmap;
1236                 }
1237
1238                 /* Now userfault register and we can use the memory */
1239                 for (i = 0; i < memory->nregions; i++) {
1240 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
1241                         reg = &dev->mem->regions[i];
1242                         struct uffdio_register reg_struct;
1243
1244                         /*
1245                          * Let's register all the mmap'ed area to ensure
1246                          * alignment on page boundary.
1247                          */
1248                         reg_struct.range.start =
1249                                 (uint64_t)(uintptr_t)reg->mmap_addr;
1250                         reg_struct.range.len = reg->mmap_size;
1251                         reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
1252
1253                         if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER,
1254                                                 &reg_struct)) {
1255                                 VHOST_LOG_CONFIG(ERR,
1256                                         "Failed to register ufd for region %d: (ufd = %d) %s\n",
1257                                         i, dev->postcopy_ufd,
1258                                         strerror(errno));
1259                                 goto err_mmap;
1260                         }
1261                         VHOST_LOG_CONFIG(INFO,
1262                                 "\t userfaultfd registered for range : "
1263                                 "%" PRIx64 " - %" PRIx64 "\n",
1264                                 (uint64_t)reg_struct.range.start,
1265                                 (uint64_t)reg_struct.range.start +
1266                                 (uint64_t)reg_struct.range.len - 1);
1267 #else
1268                         goto err_mmap;
1269 #endif
1270                 }
1271         }
1272
1273         for (i = 0; i < dev->nr_vring; i++) {
1274                 struct vhost_virtqueue *vq = dev->virtqueue[i];
1275
1276                 if (vq->desc || vq->avail || vq->used) {
1277                         /*
1278                          * If the memory table got updated, the ring addresses
1279                          * need to be translated again as virtual addresses have
1280                          * changed.
1281                          */
1282                         vring_invalidate(dev, vq);
1283
1284                         dev = translate_ring_addresses(dev, i);
1285                         if (!dev) {
1286                                 dev = *pdev;
1287                                 goto err_mmap;
1288                         }
1289
1290                         *pdev = dev;
1291                 }
1292         }
1293
1294         dump_guest_pages(dev);
1295
1296         return RTE_VHOST_MSG_RESULT_OK;
1297
1298 err_mmap:
1299         free_mem_region(dev);
1300         rte_free(dev->mem);
1301         dev->mem = NULL;
1302         return RTE_VHOST_MSG_RESULT_ERR;
1303 }
1304
1305 static bool
1306 vq_is_ready(struct virtio_net *dev, struct vhost_virtqueue *vq)
1307 {
1308         bool rings_ok;
1309
1310         if (!vq)
1311                 return false;
1312
1313         if (vq_is_packed(dev))
1314                 rings_ok = vq->desc_packed && vq->driver_event &&
1315                         vq->device_event;
1316         else
1317                 rings_ok = vq->desc && vq->avail && vq->used;
1318
1319         return rings_ok &&
1320                vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1321                vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD &&
1322                vq->enabled;
1323 }
1324
1325 #define VIRTIO_DEV_NUM_VQS_TO_BE_READY 2u
1326
1327 static int
1328 virtio_is_ready(struct virtio_net *dev)
1329 {
1330         struct vhost_virtqueue *vq;
1331         uint32_t i;
1332
1333         if (dev->nr_vring < VIRTIO_DEV_NUM_VQS_TO_BE_READY)
1334                 return 0;
1335
1336         for (i = 0; i < VIRTIO_DEV_NUM_VQS_TO_BE_READY; i++) {
1337                 vq = dev->virtqueue[i];
1338
1339                 if (!vq_is_ready(dev, vq))
1340                         return 0;
1341         }
1342
1343         if (!(dev->flags & VIRTIO_DEV_RUNNING))
1344                 VHOST_LOG_CONFIG(INFO,
1345                         "virtio is now ready for processing.\n");
1346         return 1;
1347 }
1348
1349 static void *
1350 inflight_mem_alloc(const char *name, size_t size, int *fd)
1351 {
1352         void *ptr;
1353         int mfd = -1;
1354         char fname[20] = "/tmp/memfd-XXXXXX";
1355
1356         *fd = -1;
1357 #ifdef MEMFD_SUPPORTED
1358         mfd = memfd_create(name, MFD_CLOEXEC);
1359 #else
1360         RTE_SET_USED(name);
1361 #endif
1362         if (mfd == -1) {
1363                 mfd = mkstemp(fname);
1364                 if (mfd == -1) {
1365                         VHOST_LOG_CONFIG(ERR,
1366                                 "failed to get inflight buffer fd\n");
1367                         return NULL;
1368                 }
1369
1370                 unlink(fname);
1371         }
1372
1373         if (ftruncate(mfd, size) == -1) {
1374                 VHOST_LOG_CONFIG(ERR,
1375                         "failed to alloc inflight buffer\n");
1376                 close(mfd);
1377                 return NULL;
1378         }
1379
1380         ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
1381         if (ptr == MAP_FAILED) {
1382                 VHOST_LOG_CONFIG(ERR,
1383                         "failed to mmap inflight buffer\n");
1384                 close(mfd);
1385                 return NULL;
1386         }
1387
1388         *fd = mfd;
1389         return ptr;
1390 }
1391
1392 static uint32_t
1393 get_pervq_shm_size_split(uint16_t queue_size)
1394 {
1395         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_split) *
1396                                   queue_size + sizeof(uint64_t) +
1397                                   sizeof(uint16_t) * 4, INFLIGHT_ALIGNMENT);
1398 }
1399
1400 static uint32_t
1401 get_pervq_shm_size_packed(uint16_t queue_size)
1402 {
1403         return RTE_ALIGN_MUL_CEIL(sizeof(struct rte_vhost_inflight_desc_packed)
1404                                   * queue_size + sizeof(uint64_t) +
1405                                   sizeof(uint16_t) * 6 + sizeof(uint8_t) * 9,
1406                                   INFLIGHT_ALIGNMENT);
1407 }
1408
1409 static int
1410 vhost_user_get_inflight_fd(struct virtio_net **pdev,
1411                            VhostUserMsg *msg,
1412                            int main_fd __rte_unused)
1413 {
1414         struct rte_vhost_inflight_info_packed *inflight_packed;
1415         uint64_t pervq_inflight_size, mmap_size;
1416         uint16_t num_queues, queue_size;
1417         struct virtio_net *dev = *pdev;
1418         int fd, i, j;
1419         void *addr;
1420
1421         if (msg->size != sizeof(msg->payload.inflight)) {
1422                 VHOST_LOG_CONFIG(ERR,
1423                         "invalid get_inflight_fd message size is %d\n",
1424                         msg->size);
1425                 return RTE_VHOST_MSG_RESULT_ERR;
1426         }
1427
1428         if (dev->inflight_info == NULL) {
1429                 dev->inflight_info = calloc(1,
1430                                             sizeof(struct inflight_mem_info));
1431                 if (!dev->inflight_info) {
1432                         VHOST_LOG_CONFIG(ERR,
1433                                 "failed to alloc dev inflight area\n");
1434                         return RTE_VHOST_MSG_RESULT_ERR;
1435                 }
1436                 dev->inflight_info->fd = -1;
1437         }
1438
1439         num_queues = msg->payload.inflight.num_queues;
1440         queue_size = msg->payload.inflight.queue_size;
1441
1442         VHOST_LOG_CONFIG(INFO, "get_inflight_fd num_queues: %u\n",
1443                 msg->payload.inflight.num_queues);
1444         VHOST_LOG_CONFIG(INFO, "get_inflight_fd queue_size: %u\n",
1445                 msg->payload.inflight.queue_size);
1446
1447         if (vq_is_packed(dev))
1448                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1449         else
1450                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1451
1452         mmap_size = num_queues * pervq_inflight_size;
1453         addr = inflight_mem_alloc("vhost-inflight", mmap_size, &fd);
1454         if (!addr) {
1455                 VHOST_LOG_CONFIG(ERR,
1456                         "failed to alloc vhost inflight area\n");
1457                         msg->payload.inflight.mmap_size = 0;
1458                 return RTE_VHOST_MSG_RESULT_ERR;
1459         }
1460         memset(addr, 0, mmap_size);
1461
1462         if (dev->inflight_info->addr) {
1463                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1464                 dev->inflight_info->addr = NULL;
1465         }
1466
1467         if (dev->inflight_info->fd >= 0) {
1468                 close(dev->inflight_info->fd);
1469                 dev->inflight_info->fd = -1;
1470         }
1471
1472         dev->inflight_info->addr = addr;
1473         dev->inflight_info->size = msg->payload.inflight.mmap_size = mmap_size;
1474         dev->inflight_info->fd = msg->fds[0] = fd;
1475         msg->payload.inflight.mmap_offset = 0;
1476         msg->fd_num = 1;
1477
1478         if (vq_is_packed(dev)) {
1479                 for (i = 0; i < num_queues; i++) {
1480                         inflight_packed =
1481                                 (struct rte_vhost_inflight_info_packed *)addr;
1482                         inflight_packed->used_wrap_counter = 1;
1483                         inflight_packed->old_used_wrap_counter = 1;
1484                         for (j = 0; j < queue_size; j++)
1485                                 inflight_packed->desc[j].next = j + 1;
1486                         addr = (void *)((char *)addr + pervq_inflight_size);
1487                 }
1488         }
1489
1490         VHOST_LOG_CONFIG(INFO,
1491                 "send inflight mmap_size: %"PRIu64"\n",
1492                 msg->payload.inflight.mmap_size);
1493         VHOST_LOG_CONFIG(INFO,
1494                 "send inflight mmap_offset: %"PRIu64"\n",
1495                 msg->payload.inflight.mmap_offset);
1496         VHOST_LOG_CONFIG(INFO,
1497                 "send inflight fd: %d\n", msg->fds[0]);
1498
1499         return RTE_VHOST_MSG_RESULT_REPLY;
1500 }
1501
1502 static int
1503 vhost_user_set_inflight_fd(struct virtio_net **pdev, VhostUserMsg *msg,
1504                            int main_fd __rte_unused)
1505 {
1506         uint64_t mmap_size, mmap_offset;
1507         uint16_t num_queues, queue_size;
1508         struct virtio_net *dev = *pdev;
1509         uint32_t pervq_inflight_size;
1510         struct vhost_virtqueue *vq;
1511         void *addr;
1512         int fd, i;
1513
1514         fd = msg->fds[0];
1515         if (msg->size != sizeof(msg->payload.inflight) || fd < 0) {
1516                 VHOST_LOG_CONFIG(ERR,
1517                         "invalid set_inflight_fd message size is %d,fd is %d\n",
1518                         msg->size, fd);
1519                 return RTE_VHOST_MSG_RESULT_ERR;
1520         }
1521
1522         mmap_size = msg->payload.inflight.mmap_size;
1523         mmap_offset = msg->payload.inflight.mmap_offset;
1524         num_queues = msg->payload.inflight.num_queues;
1525         queue_size = msg->payload.inflight.queue_size;
1526
1527         if (vq_is_packed(dev))
1528                 pervq_inflight_size = get_pervq_shm_size_packed(queue_size);
1529         else
1530                 pervq_inflight_size = get_pervq_shm_size_split(queue_size);
1531
1532         VHOST_LOG_CONFIG(INFO,
1533                 "set_inflight_fd mmap_size: %"PRIu64"\n", mmap_size);
1534         VHOST_LOG_CONFIG(INFO,
1535                 "set_inflight_fd mmap_offset: %"PRIu64"\n", mmap_offset);
1536         VHOST_LOG_CONFIG(INFO,
1537                 "set_inflight_fd num_queues: %u\n", num_queues);
1538         VHOST_LOG_CONFIG(INFO,
1539                 "set_inflight_fd queue_size: %u\n", queue_size);
1540         VHOST_LOG_CONFIG(INFO,
1541                 "set_inflight_fd fd: %d\n", fd);
1542         VHOST_LOG_CONFIG(INFO,
1543                 "set_inflight_fd pervq_inflight_size: %d\n",
1544                 pervq_inflight_size);
1545
1546         if (!dev->inflight_info) {
1547                 dev->inflight_info = calloc(1,
1548                                             sizeof(struct inflight_mem_info));
1549                 if (dev->inflight_info == NULL) {
1550                         VHOST_LOG_CONFIG(ERR,
1551                                 "failed to alloc dev inflight area\n");
1552                         return RTE_VHOST_MSG_RESULT_ERR;
1553                 }
1554                 dev->inflight_info->fd = -1;
1555         }
1556
1557         if (dev->inflight_info->addr) {
1558                 munmap(dev->inflight_info->addr, dev->inflight_info->size);
1559                 dev->inflight_info->addr = NULL;
1560         }
1561
1562         addr = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1563                     fd, mmap_offset);
1564         if (addr == MAP_FAILED) {
1565                 VHOST_LOG_CONFIG(ERR, "failed to mmap share memory.\n");
1566                 return RTE_VHOST_MSG_RESULT_ERR;
1567         }
1568
1569         if (dev->inflight_info->fd >= 0) {
1570                 close(dev->inflight_info->fd);
1571                 dev->inflight_info->fd = -1;
1572         }
1573
1574         dev->inflight_info->fd = fd;
1575         dev->inflight_info->addr = addr;
1576         dev->inflight_info->size = mmap_size;
1577
1578         for (i = 0; i < num_queues; i++) {
1579                 vq = dev->virtqueue[i];
1580                 if (vq_is_packed(dev)) {
1581                         vq->inflight_packed = addr;
1582                         vq->inflight_packed->desc_num = queue_size;
1583                 } else {
1584                         vq->inflight_split = addr;
1585                         vq->inflight_split->desc_num = queue_size;
1586                 }
1587                 addr = (void *)((char *)addr + pervq_inflight_size);
1588         }
1589
1590         return RTE_VHOST_MSG_RESULT_OK;
1591 }
1592
1593 static int
1594 vhost_user_set_vring_call(struct virtio_net **pdev, struct VhostUserMsg *msg,
1595                         int main_fd __rte_unused)
1596 {
1597         struct virtio_net *dev = *pdev;
1598         struct vhost_vring_file file;
1599         struct vhost_virtqueue *vq;
1600         int expected_fds;
1601
1602         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1603         if (validate_msg_fds(msg, expected_fds) != 0)
1604                 return RTE_VHOST_MSG_RESULT_ERR;
1605
1606         file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1607         if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1608                 file.fd = VIRTIO_INVALID_EVENTFD;
1609         else
1610                 file.fd = msg->fds[0];
1611         VHOST_LOG_CONFIG(INFO,
1612                 "vring call idx:%d file:%d\n", file.index, file.fd);
1613
1614         vq = dev->virtqueue[file.index];
1615         if (vq->callfd >= 0)
1616                 close(vq->callfd);
1617
1618         vq->callfd = file.fd;
1619
1620         return RTE_VHOST_MSG_RESULT_OK;
1621 }
1622
1623 static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
1624                         struct VhostUserMsg *msg,
1625                         int main_fd __rte_unused)
1626 {
1627         int expected_fds;
1628
1629         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1630         if (validate_msg_fds(msg, expected_fds) != 0)
1631                 return RTE_VHOST_MSG_RESULT_ERR;
1632
1633         if (!(msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1634                 close(msg->fds[0]);
1635         VHOST_LOG_CONFIG(INFO, "not implemented\n");
1636
1637         return RTE_VHOST_MSG_RESULT_OK;
1638 }
1639
1640 static int
1641 resubmit_desc_compare(const void *a, const void *b)
1642 {
1643         const struct rte_vhost_resubmit_desc *desc0 = a;
1644         const struct rte_vhost_resubmit_desc *desc1 = b;
1645
1646         if (desc1->counter > desc0->counter)
1647                 return 1;
1648
1649         return -1;
1650 }
1651
1652 static int
1653 vhost_check_queue_inflights_split(struct virtio_net *dev,
1654                                   struct vhost_virtqueue *vq)
1655 {
1656         uint16_t i;
1657         uint16_t resubmit_num = 0, last_io, num;
1658         struct vring_used *used = vq->used;
1659         struct rte_vhost_resubmit_info *resubmit;
1660         struct rte_vhost_inflight_info_split *inflight_split;
1661
1662         if (!(dev->protocol_features &
1663             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1664                 return RTE_VHOST_MSG_RESULT_OK;
1665
1666         /* The frontend may still not support the inflight feature
1667          * although we negotiate the protocol feature.
1668          */
1669         if ((!vq->inflight_split))
1670                 return RTE_VHOST_MSG_RESULT_OK;
1671
1672         if (!vq->inflight_split->version) {
1673                 vq->inflight_split->version = INFLIGHT_VERSION;
1674                 return RTE_VHOST_MSG_RESULT_OK;
1675         }
1676
1677         if (vq->resubmit_inflight)
1678                 return RTE_VHOST_MSG_RESULT_OK;
1679
1680         inflight_split = vq->inflight_split;
1681         vq->global_counter = 0;
1682         last_io = inflight_split->last_inflight_io;
1683
1684         if (inflight_split->used_idx != used->idx) {
1685                 inflight_split->desc[last_io].inflight = 0;
1686                 rte_smp_mb();
1687                 inflight_split->used_idx = used->idx;
1688         }
1689
1690         for (i = 0; i < inflight_split->desc_num; i++) {
1691                 if (inflight_split->desc[i].inflight == 1)
1692                         resubmit_num++;
1693         }
1694
1695         vq->last_avail_idx += resubmit_num;
1696
1697         if (resubmit_num) {
1698                 resubmit  = calloc(1, sizeof(struct rte_vhost_resubmit_info));
1699                 if (!resubmit) {
1700                         VHOST_LOG_CONFIG(ERR,
1701                                 "failed to allocate memory for resubmit info.\n");
1702                         return RTE_VHOST_MSG_RESULT_ERR;
1703                 }
1704
1705                 resubmit->resubmit_list = calloc(resubmit_num,
1706                         sizeof(struct rte_vhost_resubmit_desc));
1707                 if (!resubmit->resubmit_list) {
1708                         VHOST_LOG_CONFIG(ERR,
1709                                 "failed to allocate memory for inflight desc.\n");
1710                         free(resubmit);
1711                         return RTE_VHOST_MSG_RESULT_ERR;
1712                 }
1713
1714                 num = 0;
1715                 for (i = 0; i < vq->inflight_split->desc_num; i++) {
1716                         if (vq->inflight_split->desc[i].inflight == 1) {
1717                                 resubmit->resubmit_list[num].index = i;
1718                                 resubmit->resubmit_list[num].counter =
1719                                         inflight_split->desc[i].counter;
1720                                 num++;
1721                         }
1722                 }
1723                 resubmit->resubmit_num = num;
1724
1725                 if (resubmit->resubmit_num > 1)
1726                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1727                               sizeof(struct rte_vhost_resubmit_desc),
1728                               resubmit_desc_compare);
1729
1730                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1731                 vq->resubmit_inflight = resubmit;
1732         }
1733
1734         return RTE_VHOST_MSG_RESULT_OK;
1735 }
1736
1737 static int
1738 vhost_check_queue_inflights_packed(struct virtio_net *dev,
1739                                    struct vhost_virtqueue *vq)
1740 {
1741         uint16_t i;
1742         uint16_t resubmit_num = 0, old_used_idx, num;
1743         struct rte_vhost_resubmit_info *resubmit;
1744         struct rte_vhost_inflight_info_packed *inflight_packed;
1745
1746         if (!(dev->protocol_features &
1747             (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
1748                 return RTE_VHOST_MSG_RESULT_OK;
1749
1750         /* The frontend may still not support the inflight feature
1751          * although we negotiate the protocol feature.
1752          */
1753         if ((!vq->inflight_packed))
1754                 return RTE_VHOST_MSG_RESULT_OK;
1755
1756         if (!vq->inflight_packed->version) {
1757                 vq->inflight_packed->version = INFLIGHT_VERSION;
1758                 return RTE_VHOST_MSG_RESULT_OK;
1759         }
1760
1761         if (vq->resubmit_inflight)
1762                 return RTE_VHOST_MSG_RESULT_OK;
1763
1764         inflight_packed = vq->inflight_packed;
1765         vq->global_counter = 0;
1766         old_used_idx = inflight_packed->old_used_idx;
1767
1768         if (inflight_packed->used_idx != old_used_idx) {
1769                 if (inflight_packed->desc[old_used_idx].inflight == 0) {
1770                         inflight_packed->old_used_idx =
1771                                 inflight_packed->used_idx;
1772                         inflight_packed->old_used_wrap_counter =
1773                                 inflight_packed->used_wrap_counter;
1774                         inflight_packed->old_free_head =
1775                                 inflight_packed->free_head;
1776                 } else {
1777                         inflight_packed->used_idx =
1778                                 inflight_packed->old_used_idx;
1779                         inflight_packed->used_wrap_counter =
1780                                 inflight_packed->old_used_wrap_counter;
1781                         inflight_packed->free_head =
1782                                 inflight_packed->old_free_head;
1783                 }
1784         }
1785
1786         for (i = 0; i < inflight_packed->desc_num; i++) {
1787                 if (inflight_packed->desc[i].inflight == 1)
1788                         resubmit_num++;
1789         }
1790
1791         if (resubmit_num) {
1792                 resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
1793                 if (resubmit == NULL) {
1794                         VHOST_LOG_CONFIG(ERR,
1795                                 "failed to allocate memory for resubmit info.\n");
1796                         return RTE_VHOST_MSG_RESULT_ERR;
1797                 }
1798
1799                 resubmit->resubmit_list = calloc(resubmit_num,
1800                         sizeof(struct rte_vhost_resubmit_desc));
1801                 if (resubmit->resubmit_list == NULL) {
1802                         VHOST_LOG_CONFIG(ERR,
1803                                 "failed to allocate memory for resubmit desc.\n");
1804                         free(resubmit);
1805                         return RTE_VHOST_MSG_RESULT_ERR;
1806                 }
1807
1808                 num = 0;
1809                 for (i = 0; i < inflight_packed->desc_num; i++) {
1810                         if (vq->inflight_packed->desc[i].inflight == 1) {
1811                                 resubmit->resubmit_list[num].index = i;
1812                                 resubmit->resubmit_list[num].counter =
1813                                         inflight_packed->desc[i].counter;
1814                                 num++;
1815                         }
1816                 }
1817                 resubmit->resubmit_num = num;
1818
1819                 if (resubmit->resubmit_num > 1)
1820                         qsort(resubmit->resubmit_list, resubmit->resubmit_num,
1821                               sizeof(struct rte_vhost_resubmit_desc),
1822                               resubmit_desc_compare);
1823
1824                 vq->global_counter = resubmit->resubmit_list[0].counter + 1;
1825                 vq->resubmit_inflight = resubmit;
1826         }
1827
1828         return RTE_VHOST_MSG_RESULT_OK;
1829 }
1830
1831 static int
1832 vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,
1833                         int main_fd __rte_unused)
1834 {
1835         struct virtio_net *dev = *pdev;
1836         struct vhost_vring_file file;
1837         struct vhost_virtqueue *vq;
1838         int expected_fds;
1839
1840         expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;
1841         if (validate_msg_fds(msg, expected_fds) != 0)
1842                 return RTE_VHOST_MSG_RESULT_ERR;
1843
1844         file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1845         if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
1846                 file.fd = VIRTIO_INVALID_EVENTFD;
1847         else
1848                 file.fd = msg->fds[0];
1849         VHOST_LOG_CONFIG(INFO,
1850                 "vring kick idx:%d file:%d\n", file.index, file.fd);
1851
1852         /* Interpret ring addresses only when ring is started. */
1853         dev = translate_ring_addresses(dev, file.index);
1854         if (!dev)
1855                 return RTE_VHOST_MSG_RESULT_ERR;
1856
1857         *pdev = dev;
1858
1859         vq = dev->virtqueue[file.index];
1860
1861         /*
1862          * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated,
1863          * the ring starts already enabled. Otherwise, it is enabled via
1864          * the SET_VRING_ENABLE message.
1865          */
1866         if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) {
1867                 vq->enabled = 1;
1868                 if (dev->notify_ops->vring_state_changed)
1869                         dev->notify_ops->vring_state_changed(
1870                                 dev->vid, file.index, 1);
1871         }
1872
1873         if (vq->kickfd >= 0)
1874                 close(vq->kickfd);
1875         vq->kickfd = file.fd;
1876
1877         if (vq_is_packed(dev)) {
1878                 if (vhost_check_queue_inflights_packed(dev, vq)) {
1879                         VHOST_LOG_CONFIG(ERR,
1880                                 "failed to inflights for vq: %d\n", file.index);
1881                         return RTE_VHOST_MSG_RESULT_ERR;
1882                 }
1883         } else {
1884                 if (vhost_check_queue_inflights_split(dev, vq)) {
1885                         VHOST_LOG_CONFIG(ERR,
1886                                 "failed to inflights for vq: %d\n", file.index);
1887                         return RTE_VHOST_MSG_RESULT_ERR;
1888                 }
1889         }
1890
1891         return RTE_VHOST_MSG_RESULT_OK;
1892 }
1893
1894 static void
1895 free_zmbufs(struct vhost_virtqueue *vq)
1896 {
1897         drain_zmbuf_list(vq);
1898
1899         rte_free(vq->zmbufs);
1900 }
1901
1902 /*
1903  * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
1904  */
1905 static int
1906 vhost_user_get_vring_base(struct virtio_net **pdev,
1907                         struct VhostUserMsg *msg,
1908                         int main_fd __rte_unused)
1909 {
1910         struct virtio_net *dev = *pdev;
1911         struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
1912         uint64_t val;
1913
1914         if (validate_msg_fds(msg, 0) != 0)
1915                 return RTE_VHOST_MSG_RESULT_ERR;
1916
1917         /* We have to stop the queue (virtio) if it is running. */
1918         vhost_destroy_device_notify(dev);
1919
1920         dev->flags &= ~VIRTIO_DEV_READY;
1921         dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
1922
1923         /* Here we are safe to get the indexes */
1924         if (vq_is_packed(dev)) {
1925                 /*
1926                  * Bit[0:14]: avail index
1927                  * Bit[15]: avail wrap counter
1928                  */
1929                 val = vq->last_avail_idx & 0x7fff;
1930                 val |= vq->avail_wrap_counter << 15;
1931                 msg->payload.state.num = val;
1932         } else {
1933                 msg->payload.state.num = vq->last_avail_idx;
1934         }
1935
1936         VHOST_LOG_CONFIG(INFO,
1937                 "vring base idx:%d file:%d\n", msg->payload.state.index,
1938                 msg->payload.state.num);
1939         /*
1940          * Based on current qemu vhost-user implementation, this message is
1941          * sent and only sent in vhost_vring_stop.
1942          * TODO: cleanup the vring, it isn't usable since here.
1943          */
1944         if (vq->kickfd >= 0)
1945                 close(vq->kickfd);
1946
1947         vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
1948
1949         if (vq->callfd >= 0)
1950                 close(vq->callfd);
1951
1952         vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
1953
1954         vq->signalled_used_valid = false;
1955
1956         if (dev->dequeue_zero_copy)
1957                 free_zmbufs(vq);
1958         if (vq_is_packed(dev)) {
1959                 rte_free(vq->shadow_used_packed);
1960                 vq->shadow_used_packed = NULL;
1961         } else {
1962                 rte_free(vq->shadow_used_split);
1963                 vq->shadow_used_split = NULL;
1964         }
1965
1966         rte_free(vq->batch_copy_elems);
1967         vq->batch_copy_elems = NULL;
1968
1969         msg->size = sizeof(msg->payload.state);
1970         msg->fd_num = 0;
1971
1972         vring_invalidate(dev, vq);
1973
1974         return RTE_VHOST_MSG_RESULT_REPLY;
1975 }
1976
1977 /*
1978  * when virtio queues are ready to work, qemu will send us to
1979  * enable the virtio queue pair.
1980  */
1981 static int
1982 vhost_user_set_vring_enable(struct virtio_net **pdev,
1983                         struct VhostUserMsg *msg,
1984                         int main_fd __rte_unused)
1985 {
1986         struct virtio_net *dev = *pdev;
1987         int enable = (int)msg->payload.state.num;
1988         int index = (int)msg->payload.state.index;
1989
1990         if (validate_msg_fds(msg, 0) != 0)
1991                 return RTE_VHOST_MSG_RESULT_ERR;
1992
1993         VHOST_LOG_CONFIG(INFO,
1994                 "set queue enable: %d to qp idx: %d\n",
1995                 enable, index);
1996
1997         /* On disable, rings have to be stopped being processed. */
1998         if (!enable && dev->dequeue_zero_copy)
1999                 drain_zmbuf_list(dev->virtqueue[index]);
2000
2001         dev->virtqueue[index]->enabled = enable;
2002
2003         return RTE_VHOST_MSG_RESULT_OK;
2004 }
2005
2006 static int
2007 vhost_user_get_protocol_features(struct virtio_net **pdev,
2008                         struct VhostUserMsg *msg,
2009                         int main_fd __rte_unused)
2010 {
2011         struct virtio_net *dev = *pdev;
2012         uint64_t features, protocol_features;
2013
2014         if (validate_msg_fds(msg, 0) != 0)
2015                 return RTE_VHOST_MSG_RESULT_ERR;
2016
2017         rte_vhost_driver_get_features(dev->ifname, &features);
2018         rte_vhost_driver_get_protocol_features(dev->ifname, &protocol_features);
2019
2020         msg->payload.u64 = protocol_features;
2021         msg->size = sizeof(msg->payload.u64);
2022         msg->fd_num = 0;
2023
2024         return RTE_VHOST_MSG_RESULT_REPLY;
2025 }
2026
2027 static int
2028 vhost_user_set_protocol_features(struct virtio_net **pdev,
2029                         struct VhostUserMsg *msg,
2030                         int main_fd __rte_unused)
2031 {
2032         struct virtio_net *dev = *pdev;
2033         uint64_t protocol_features = msg->payload.u64;
2034         uint64_t slave_protocol_features = 0;
2035
2036         if (validate_msg_fds(msg, 0) != 0)
2037                 return RTE_VHOST_MSG_RESULT_ERR;
2038
2039         rte_vhost_driver_get_protocol_features(dev->ifname,
2040                         &slave_protocol_features);
2041         if (protocol_features & ~slave_protocol_features) {
2042                 VHOST_LOG_CONFIG(ERR,
2043                         "(%d) received invalid protocol features.\n",
2044                         dev->vid);
2045                 return RTE_VHOST_MSG_RESULT_ERR;
2046         }
2047
2048         dev->protocol_features = protocol_features;
2049         VHOST_LOG_CONFIG(INFO,
2050                 "negotiated Vhost-user protocol features: 0x%" PRIx64 "\n",
2051                 dev->protocol_features);
2052
2053         return RTE_VHOST_MSG_RESULT_OK;
2054 }
2055
2056 static int
2057 vhost_user_set_log_base(struct virtio_net **pdev, struct VhostUserMsg *msg,
2058                         int main_fd __rte_unused)
2059 {
2060         struct virtio_net *dev = *pdev;
2061         int fd = msg->fds[0];
2062         uint64_t size, off;
2063         void *addr;
2064
2065         if (validate_msg_fds(msg, 1) != 0)
2066                 return RTE_VHOST_MSG_RESULT_ERR;
2067
2068         if (fd < 0) {
2069                 VHOST_LOG_CONFIG(ERR, "invalid log fd: %d\n", fd);
2070                 return RTE_VHOST_MSG_RESULT_ERR;
2071         }
2072
2073         if (msg->size != sizeof(VhostUserLog)) {
2074                 VHOST_LOG_CONFIG(ERR,
2075                         "invalid log base msg size: %"PRId32" != %d\n",
2076                         msg->size, (int)sizeof(VhostUserLog));
2077                 return RTE_VHOST_MSG_RESULT_ERR;
2078         }
2079
2080         size = msg->payload.log.mmap_size;
2081         off  = msg->payload.log.mmap_offset;
2082
2083         /* Check for mmap size and offset overflow. */
2084         if (off >= -size) {
2085                 VHOST_LOG_CONFIG(ERR,
2086                         "log offset %#"PRIx64" and log size %#"PRIx64" overflow\n",
2087                         off, size);
2088                 return RTE_VHOST_MSG_RESULT_ERR;
2089         }
2090
2091         VHOST_LOG_CONFIG(INFO,
2092                 "log mmap size: %"PRId64", offset: %"PRId64"\n",
2093                 size, off);
2094
2095         /*
2096          * mmap from 0 to workaround a hugepage mmap bug: mmap will
2097          * fail when offset is not page size aligned.
2098          */
2099         addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
2100         close(fd);
2101         if (addr == MAP_FAILED) {
2102                 VHOST_LOG_CONFIG(ERR, "mmap log base failed!\n");
2103                 return RTE_VHOST_MSG_RESULT_ERR;
2104         }
2105
2106         /*
2107          * Free previously mapped log memory on occasionally
2108          * multiple VHOST_USER_SET_LOG_BASE.
2109          */
2110         if (dev->log_addr) {
2111                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
2112         }
2113         dev->log_addr = (uint64_t)(uintptr_t)addr;
2114         dev->log_base = dev->log_addr + off;
2115         dev->log_size = size;
2116
2117         /*
2118          * The spec is not clear about it (yet), but QEMU doesn't expect
2119          * any payload in the reply.
2120          */
2121         msg->size = 0;
2122         msg->fd_num = 0;
2123
2124         return RTE_VHOST_MSG_RESULT_REPLY;
2125 }
2126
2127 static int vhost_user_set_log_fd(struct virtio_net **pdev __rte_unused,
2128                         struct VhostUserMsg *msg,
2129                         int main_fd __rte_unused)
2130 {
2131         if (validate_msg_fds(msg, 1) != 0)
2132                 return RTE_VHOST_MSG_RESULT_ERR;
2133
2134         close(msg->fds[0]);
2135         VHOST_LOG_CONFIG(INFO, "not implemented.\n");
2136
2137         return RTE_VHOST_MSG_RESULT_OK;
2138 }
2139
2140 /*
2141  * An rarp packet is constructed and broadcasted to notify switches about
2142  * the new location of the migrated VM, so that packets from outside will
2143  * not be lost after migration.
2144  *
2145  * However, we don't actually "send" a rarp packet here, instead, we set
2146  * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
2147  */
2148 static int
2149 vhost_user_send_rarp(struct virtio_net **pdev, struct VhostUserMsg *msg,
2150                         int main_fd __rte_unused)
2151 {
2152         struct virtio_net *dev = *pdev;
2153         uint8_t *mac = (uint8_t *)&msg->payload.u64;
2154         struct rte_vdpa_device *vdpa_dev;
2155
2156         if (validate_msg_fds(msg, 0) != 0)
2157                 return RTE_VHOST_MSG_RESULT_ERR;
2158
2159         VHOST_LOG_CONFIG(DEBUG,
2160                 ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
2161                 mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
2162         memcpy(dev->mac.addr_bytes, mac, 6);
2163
2164         /*
2165          * Set the flag to inject a RARP broadcast packet at
2166          * rte_vhost_dequeue_burst().
2167          *
2168          * __ATOMIC_RELEASE ordering is for making sure the mac is
2169          * copied before the flag is set.
2170          */
2171         __atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
2172         vdpa_dev = dev->vdpa_dev;
2173         if (vdpa_dev && vdpa_dev->ops->migration_done)
2174                 vdpa_dev->ops->migration_done(dev->vid);
2175
2176         return RTE_VHOST_MSG_RESULT_OK;
2177 }
2178
2179 static int
2180 vhost_user_net_set_mtu(struct virtio_net **pdev, struct VhostUserMsg *msg,
2181                         int main_fd __rte_unused)
2182 {
2183         struct virtio_net *dev = *pdev;
2184
2185         if (validate_msg_fds(msg, 0) != 0)
2186                 return RTE_VHOST_MSG_RESULT_ERR;
2187
2188         if (msg->payload.u64 < VIRTIO_MIN_MTU ||
2189                         msg->payload.u64 > VIRTIO_MAX_MTU) {
2190                 VHOST_LOG_CONFIG(ERR, "Invalid MTU size (%"PRIu64")\n",
2191                                 msg->payload.u64);
2192
2193                 return RTE_VHOST_MSG_RESULT_ERR;
2194         }
2195
2196         dev->mtu = msg->payload.u64;
2197
2198         return RTE_VHOST_MSG_RESULT_OK;
2199 }
2200
2201 static int
2202 vhost_user_set_req_fd(struct virtio_net **pdev, struct VhostUserMsg *msg,
2203                         int main_fd __rte_unused)
2204 {
2205         struct virtio_net *dev = *pdev;
2206         int fd = msg->fds[0];
2207
2208         if (validate_msg_fds(msg, 1) != 0)
2209                 return RTE_VHOST_MSG_RESULT_ERR;
2210
2211         if (fd < 0) {
2212                 VHOST_LOG_CONFIG(ERR,
2213                                 "Invalid file descriptor for slave channel (%d)\n",
2214                                 fd);
2215                 return RTE_VHOST_MSG_RESULT_ERR;
2216         }
2217
2218         if (dev->slave_req_fd >= 0)
2219                 close(dev->slave_req_fd);
2220
2221         dev->slave_req_fd = fd;
2222
2223         return RTE_VHOST_MSG_RESULT_OK;
2224 }
2225
2226 static int
2227 is_vring_iotlb_split(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2228 {
2229         struct vhost_vring_addr *ra;
2230         uint64_t start, end, len;
2231
2232         start = imsg->iova;
2233         end = start + imsg->size;
2234
2235         ra = &vq->ring_addrs;
2236         len = sizeof(struct vring_desc) * vq->size;
2237         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2238                 return 1;
2239
2240         len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
2241         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2242                 return 1;
2243
2244         len = sizeof(struct vring_used) +
2245                sizeof(struct vring_used_elem) * vq->size;
2246         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2247                 return 1;
2248
2249         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2250                 len = sizeof(uint64_t);
2251                 if (ra->log_guest_addr < end &&
2252                     (ra->log_guest_addr + len) > start)
2253                         return 1;
2254         }
2255
2256         return 0;
2257 }
2258
2259 static int
2260 is_vring_iotlb_packed(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg)
2261 {
2262         struct vhost_vring_addr *ra;
2263         uint64_t start, end, len;
2264
2265         start = imsg->iova;
2266         end = start + imsg->size;
2267
2268         ra = &vq->ring_addrs;
2269         len = sizeof(struct vring_packed_desc) * vq->size;
2270         if (ra->desc_user_addr < end && (ra->desc_user_addr + len) > start)
2271                 return 1;
2272
2273         len = sizeof(struct vring_packed_desc_event);
2274         if (ra->avail_user_addr < end && (ra->avail_user_addr + len) > start)
2275                 return 1;
2276
2277         len = sizeof(struct vring_packed_desc_event);
2278         if (ra->used_user_addr < end && (ra->used_user_addr + len) > start)
2279                 return 1;
2280
2281         if (ra->flags & (1 << VHOST_VRING_F_LOG)) {
2282                 len = sizeof(uint64_t);
2283                 if (ra->log_guest_addr < end &&
2284                     (ra->log_guest_addr + len) > start)
2285                         return 1;
2286         }
2287
2288         return 0;
2289 }
2290
2291 static int is_vring_iotlb(struct virtio_net *dev,
2292                           struct vhost_virtqueue *vq,
2293                           struct vhost_iotlb_msg *imsg)
2294 {
2295         if (vq_is_packed(dev))
2296                 return is_vring_iotlb_packed(vq, imsg);
2297         else
2298                 return is_vring_iotlb_split(vq, imsg);
2299 }
2300
2301 static int
2302 vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg,
2303                         int main_fd __rte_unused)
2304 {
2305         struct virtio_net *dev = *pdev;
2306         struct vhost_iotlb_msg *imsg = &msg->payload.iotlb;
2307         uint16_t i;
2308         uint64_t vva, len;
2309
2310         if (validate_msg_fds(msg, 0) != 0)
2311                 return RTE_VHOST_MSG_RESULT_ERR;
2312
2313         switch (imsg->type) {
2314         case VHOST_IOTLB_UPDATE:
2315                 len = imsg->size;
2316                 vva = qva_to_vva(dev, imsg->uaddr, &len);
2317                 if (!vva)
2318                         return RTE_VHOST_MSG_RESULT_ERR;
2319
2320                 for (i = 0; i < dev->nr_vring; i++) {
2321                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2322
2323                         vhost_user_iotlb_cache_insert(vq, imsg->iova, vva,
2324                                         len, imsg->perm);
2325
2326                         if (is_vring_iotlb(dev, vq, imsg))
2327                                 *pdev = dev = translate_ring_addresses(dev, i);
2328                 }
2329                 break;
2330         case VHOST_IOTLB_INVALIDATE:
2331                 for (i = 0; i < dev->nr_vring; i++) {
2332                         struct vhost_virtqueue *vq = dev->virtqueue[i];
2333
2334                         vhost_user_iotlb_cache_remove(vq, imsg->iova,
2335                                         imsg->size);
2336
2337                         if (is_vring_iotlb(dev, vq, imsg))
2338                                 vring_invalidate(dev, vq);
2339                 }
2340                 break;
2341         default:
2342                 VHOST_LOG_CONFIG(ERR, "Invalid IOTLB message type (%d)\n",
2343                                 imsg->type);
2344                 return RTE_VHOST_MSG_RESULT_ERR;
2345         }
2346
2347         return RTE_VHOST_MSG_RESULT_OK;
2348 }
2349
2350 static int
2351 vhost_user_set_postcopy_advise(struct virtio_net **pdev,
2352                         struct VhostUserMsg *msg,
2353                         int main_fd __rte_unused)
2354 {
2355         struct virtio_net *dev = *pdev;
2356 #ifdef RTE_LIBRTE_VHOST_POSTCOPY
2357         struct uffdio_api api_struct;
2358
2359         if (validate_msg_fds(msg, 0) != 0)
2360                 return RTE_VHOST_MSG_RESULT_ERR;
2361
2362         dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
2363
2364         if (dev->postcopy_ufd == -1) {
2365                 VHOST_LOG_CONFIG(ERR, "Userfaultfd not available: %s\n",
2366                         strerror(errno));
2367                 return RTE_VHOST_MSG_RESULT_ERR;
2368         }
2369         api_struct.api = UFFD_API;
2370         api_struct.features = 0;
2371         if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
2372                 VHOST_LOG_CONFIG(ERR, "UFFDIO_API ioctl failure: %s\n",
2373                         strerror(errno));
2374                 close(dev->postcopy_ufd);
2375                 dev->postcopy_ufd = -1;
2376                 return RTE_VHOST_MSG_RESULT_ERR;
2377         }
2378         msg->fds[0] = dev->postcopy_ufd;
2379         msg->fd_num = 1;
2380
2381         return RTE_VHOST_MSG_RESULT_REPLY;
2382 #else
2383         dev->postcopy_ufd = -1;
2384         msg->fd_num = 0;
2385
2386         return RTE_VHOST_MSG_RESULT_ERR;
2387 #endif
2388 }
2389
2390 static int
2391 vhost_user_set_postcopy_listen(struct virtio_net **pdev,
2392                         struct VhostUserMsg *msg __rte_unused,
2393                         int main_fd __rte_unused)
2394 {
2395         struct virtio_net *dev = *pdev;
2396
2397         if (validate_msg_fds(msg, 0) != 0)
2398                 return RTE_VHOST_MSG_RESULT_ERR;
2399
2400         if (dev->mem && dev->mem->nregions) {
2401                 VHOST_LOG_CONFIG(ERR,
2402                         "Regions already registered at postcopy-listen\n");
2403                 return RTE_VHOST_MSG_RESULT_ERR;
2404         }
2405         dev->postcopy_listening = 1;
2406
2407         return RTE_VHOST_MSG_RESULT_OK;
2408 }
2409
2410 static int
2411 vhost_user_postcopy_end(struct virtio_net **pdev, struct VhostUserMsg *msg,
2412                         int main_fd __rte_unused)
2413 {
2414         struct virtio_net *dev = *pdev;
2415
2416         if (validate_msg_fds(msg, 0) != 0)
2417                 return RTE_VHOST_MSG_RESULT_ERR;
2418
2419         dev->postcopy_listening = 0;
2420         if (dev->postcopy_ufd >= 0) {
2421                 close(dev->postcopy_ufd);
2422                 dev->postcopy_ufd = -1;
2423         }
2424
2425         msg->payload.u64 = 0;
2426         msg->size = sizeof(msg->payload.u64);
2427         msg->fd_num = 0;
2428
2429         return RTE_VHOST_MSG_RESULT_REPLY;
2430 }
2431
2432 typedef int (*vhost_message_handler_t)(struct virtio_net **pdev,
2433                                         struct VhostUserMsg *msg,
2434                                         int main_fd);
2435 static vhost_message_handler_t vhost_message_handlers[VHOST_USER_MAX] = {
2436         [VHOST_USER_NONE] = NULL,
2437         [VHOST_USER_GET_FEATURES] = vhost_user_get_features,
2438         [VHOST_USER_SET_FEATURES] = vhost_user_set_features,
2439         [VHOST_USER_SET_OWNER] = vhost_user_set_owner,
2440         [VHOST_USER_RESET_OWNER] = vhost_user_reset_owner,
2441         [VHOST_USER_SET_MEM_TABLE] = vhost_user_set_mem_table,
2442         [VHOST_USER_SET_LOG_BASE] = vhost_user_set_log_base,
2443         [VHOST_USER_SET_LOG_FD] = vhost_user_set_log_fd,
2444         [VHOST_USER_SET_VRING_NUM] = vhost_user_set_vring_num,
2445         [VHOST_USER_SET_VRING_ADDR] = vhost_user_set_vring_addr,
2446         [VHOST_USER_SET_VRING_BASE] = vhost_user_set_vring_base,
2447         [VHOST_USER_GET_VRING_BASE] = vhost_user_get_vring_base,
2448         [VHOST_USER_SET_VRING_KICK] = vhost_user_set_vring_kick,
2449         [VHOST_USER_SET_VRING_CALL] = vhost_user_set_vring_call,
2450         [VHOST_USER_SET_VRING_ERR] = vhost_user_set_vring_err,
2451         [VHOST_USER_GET_PROTOCOL_FEATURES] = vhost_user_get_protocol_features,
2452         [VHOST_USER_SET_PROTOCOL_FEATURES] = vhost_user_set_protocol_features,
2453         [VHOST_USER_GET_QUEUE_NUM] = vhost_user_get_queue_num,
2454         [VHOST_USER_SET_VRING_ENABLE] = vhost_user_set_vring_enable,
2455         [VHOST_USER_SEND_RARP] = vhost_user_send_rarp,
2456         [VHOST_USER_NET_SET_MTU] = vhost_user_net_set_mtu,
2457         [VHOST_USER_SET_SLAVE_REQ_FD] = vhost_user_set_req_fd,
2458         [VHOST_USER_IOTLB_MSG] = vhost_user_iotlb_msg,
2459         [VHOST_USER_POSTCOPY_ADVISE] = vhost_user_set_postcopy_advise,
2460         [VHOST_USER_POSTCOPY_LISTEN] = vhost_user_set_postcopy_listen,
2461         [VHOST_USER_POSTCOPY_END] = vhost_user_postcopy_end,
2462         [VHOST_USER_GET_INFLIGHT_FD] = vhost_user_get_inflight_fd,
2463         [VHOST_USER_SET_INFLIGHT_FD] = vhost_user_set_inflight_fd,
2464 };
2465
2466 /* return bytes# of read on success or negative val on failure. */
2467 static int
2468 read_vhost_message(int sockfd, struct VhostUserMsg *msg)
2469 {
2470         int ret;
2471
2472         ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
2473                 msg->fds, VHOST_MEMORY_MAX_NREGIONS, &msg->fd_num);
2474         if (ret <= 0) {
2475                 return ret;
2476         } else if (ret != VHOST_USER_HDR_SIZE) {
2477                 VHOST_LOG_CONFIG(ERR, "Unexpected header size read\n");
2478                 close_msg_fds(msg);
2479                 return -1;
2480         }
2481
2482         if (msg->size) {
2483                 if (msg->size > sizeof(msg->payload)) {
2484                         VHOST_LOG_CONFIG(ERR,
2485                                 "invalid msg size: %d\n", msg->size);
2486                         return -1;
2487                 }
2488                 ret = read(sockfd, &msg->payload, msg->size);
2489                 if (ret <= 0)
2490                         return ret;
2491                 if (ret != (int)msg->size) {
2492                         VHOST_LOG_CONFIG(ERR,
2493                                 "read control message failed\n");
2494                         return -1;
2495                 }
2496         }
2497
2498         return ret;
2499 }
2500
2501 static int
2502 send_vhost_message(int sockfd, struct VhostUserMsg *msg)
2503 {
2504         if (!msg)
2505                 return 0;
2506
2507         return send_fd_message(sockfd, (char *)msg,
2508                 VHOST_USER_HDR_SIZE + msg->size, msg->fds, msg->fd_num);
2509 }
2510
2511 static int
2512 send_vhost_reply(int sockfd, struct VhostUserMsg *msg)
2513 {
2514         if (!msg)
2515                 return 0;
2516
2517         msg->flags &= ~VHOST_USER_VERSION_MASK;
2518         msg->flags &= ~VHOST_USER_NEED_REPLY;
2519         msg->flags |= VHOST_USER_VERSION;
2520         msg->flags |= VHOST_USER_REPLY_MASK;
2521
2522         return send_vhost_message(sockfd, msg);
2523 }
2524
2525 static int
2526 send_vhost_slave_message(struct virtio_net *dev, struct VhostUserMsg *msg)
2527 {
2528         int ret;
2529
2530         if (msg->flags & VHOST_USER_NEED_REPLY)
2531                 rte_spinlock_lock(&dev->slave_req_lock);
2532
2533         ret = send_vhost_message(dev->slave_req_fd, msg);
2534         if (ret < 0 && (msg->flags & VHOST_USER_NEED_REPLY))
2535                 rte_spinlock_unlock(&dev->slave_req_lock);
2536
2537         return ret;
2538 }
2539
2540 /*
2541  * Allocate a queue pair if it hasn't been allocated yet
2542  */
2543 static int
2544 vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev,
2545                         struct VhostUserMsg *msg)
2546 {
2547         uint32_t vring_idx;
2548
2549         switch (msg->request.master) {
2550         case VHOST_USER_SET_VRING_KICK:
2551         case VHOST_USER_SET_VRING_CALL:
2552         case VHOST_USER_SET_VRING_ERR:
2553                 vring_idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
2554                 break;
2555         case VHOST_USER_SET_VRING_NUM:
2556         case VHOST_USER_SET_VRING_BASE:
2557         case VHOST_USER_SET_VRING_ENABLE:
2558                 vring_idx = msg->payload.state.index;
2559                 break;
2560         case VHOST_USER_SET_VRING_ADDR:
2561                 vring_idx = msg->payload.addr.index;
2562                 break;
2563         default:
2564                 return 0;
2565         }
2566
2567         if (vring_idx >= VHOST_MAX_VRING) {
2568                 VHOST_LOG_CONFIG(ERR,
2569                         "invalid vring index: %u\n", vring_idx);
2570                 return -1;
2571         }
2572
2573         if (dev->virtqueue[vring_idx])
2574                 return 0;
2575
2576         return alloc_vring_queue(dev, vring_idx);
2577 }
2578
2579 static void
2580 vhost_user_lock_all_queue_pairs(struct virtio_net *dev)
2581 {
2582         unsigned int i = 0;
2583         unsigned int vq_num = 0;
2584
2585         while (vq_num < dev->nr_vring) {
2586                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2587
2588                 if (vq) {
2589                         rte_spinlock_lock(&vq->access_lock);
2590                         vq_num++;
2591                 }
2592                 i++;
2593         }
2594 }
2595
2596 static void
2597 vhost_user_unlock_all_queue_pairs(struct virtio_net *dev)
2598 {
2599         unsigned int i = 0;
2600         unsigned int vq_num = 0;
2601
2602         while (vq_num < dev->nr_vring) {
2603                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2604
2605                 if (vq) {
2606                         rte_spinlock_unlock(&vq->access_lock);
2607                         vq_num++;
2608                 }
2609                 i++;
2610         }
2611 }
2612
2613 int
2614 vhost_user_msg_handler(int vid, int fd)
2615 {
2616         struct virtio_net *dev;
2617         struct VhostUserMsg msg;
2618         struct rte_vdpa_device *vdpa_dev;
2619         int ret;
2620         int unlock_required = 0;
2621         bool handled;
2622         int request;
2623         uint32_t i;
2624
2625         dev = get_device(vid);
2626         if (dev == NULL)
2627                 return -1;
2628
2629         if (!dev->notify_ops) {
2630                 dev->notify_ops = vhost_driver_callback_get(dev->ifname);
2631                 if (!dev->notify_ops) {
2632                         VHOST_LOG_CONFIG(ERR,
2633                                 "failed to get callback ops for driver %s\n",
2634                                 dev->ifname);
2635                         return -1;
2636                 }
2637         }
2638
2639         ret = read_vhost_message(fd, &msg);
2640         if (ret <= 0) {
2641                 if (ret < 0)
2642                         VHOST_LOG_CONFIG(ERR,
2643                                 "vhost read message failed\n");
2644                 else
2645                         VHOST_LOG_CONFIG(INFO,
2646                                 "vhost peer closed\n");
2647
2648                 return -1;
2649         }
2650
2651         ret = 0;
2652         request = msg.request.master;
2653         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX &&
2654                         vhost_message_str[request]) {
2655                 if (request != VHOST_USER_IOTLB_MSG)
2656                         VHOST_LOG_CONFIG(INFO, "read message %s\n",
2657                                 vhost_message_str[request]);
2658                 else
2659                         VHOST_LOG_CONFIG(DEBUG, "read message %s\n",
2660                                 vhost_message_str[request]);
2661         } else {
2662                 VHOST_LOG_CONFIG(DEBUG, "External request %d\n", request);
2663         }
2664
2665         ret = vhost_user_check_and_alloc_queue_pair(dev, &msg);
2666         if (ret < 0) {
2667                 VHOST_LOG_CONFIG(ERR,
2668                         "failed to alloc queue\n");
2669                 return -1;
2670         }
2671
2672         /*
2673          * Note: we don't lock all queues on VHOST_USER_GET_VRING_BASE
2674          * and VHOST_USER_RESET_OWNER, since it is sent when virtio stops
2675          * and device is destroyed. destroy_device waits for queues to be
2676          * inactive, so it is safe. Otherwise taking the access_lock
2677          * would cause a dead lock.
2678          */
2679         switch (request) {
2680         case VHOST_USER_SET_FEATURES:
2681         case VHOST_USER_SET_PROTOCOL_FEATURES:
2682         case VHOST_USER_SET_OWNER:
2683         case VHOST_USER_SET_MEM_TABLE:
2684         case VHOST_USER_SET_LOG_BASE:
2685         case VHOST_USER_SET_LOG_FD:
2686         case VHOST_USER_SET_VRING_NUM:
2687         case VHOST_USER_SET_VRING_ADDR:
2688         case VHOST_USER_SET_VRING_BASE:
2689         case VHOST_USER_SET_VRING_KICK:
2690         case VHOST_USER_SET_VRING_CALL:
2691         case VHOST_USER_SET_VRING_ERR:
2692         case VHOST_USER_SET_VRING_ENABLE:
2693         case VHOST_USER_SEND_RARP:
2694         case VHOST_USER_NET_SET_MTU:
2695         case VHOST_USER_SET_SLAVE_REQ_FD:
2696                 if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
2697                         vhost_user_lock_all_queue_pairs(dev);
2698                         unlock_required = 1;
2699                 }
2700                 break;
2701         default:
2702                 break;
2703
2704         }
2705
2706         handled = false;
2707         if (dev->extern_ops.pre_msg_handle) {
2708                 ret = (*dev->extern_ops.pre_msg_handle)(dev->vid,
2709                                 (void *)&msg);
2710                 switch (ret) {
2711                 case RTE_VHOST_MSG_RESULT_REPLY:
2712                         send_vhost_reply(fd, &msg);
2713                         /* Fall-through */
2714                 case RTE_VHOST_MSG_RESULT_ERR:
2715                 case RTE_VHOST_MSG_RESULT_OK:
2716                         handled = true;
2717                         goto skip_to_post_handle;
2718                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
2719                 default:
2720                         break;
2721                 }
2722         }
2723
2724         if (request > VHOST_USER_NONE && request < VHOST_USER_MAX) {
2725                 if (!vhost_message_handlers[request])
2726                         goto skip_to_post_handle;
2727                 ret = vhost_message_handlers[request](&dev, &msg, fd);
2728
2729                 switch (ret) {
2730                 case RTE_VHOST_MSG_RESULT_ERR:
2731                         VHOST_LOG_CONFIG(ERR,
2732                                 "Processing %s failed.\n",
2733                                 vhost_message_str[request]);
2734                         handled = true;
2735                         break;
2736                 case RTE_VHOST_MSG_RESULT_OK:
2737                         VHOST_LOG_CONFIG(DEBUG,
2738                                 "Processing %s succeeded.\n",
2739                                 vhost_message_str[request]);
2740                         handled = true;
2741                         break;
2742                 case RTE_VHOST_MSG_RESULT_REPLY:
2743                         VHOST_LOG_CONFIG(DEBUG,
2744                                 "Processing %s succeeded and needs reply.\n",
2745                                 vhost_message_str[request]);
2746                         send_vhost_reply(fd, &msg);
2747                         handled = true;
2748                         break;
2749                 default:
2750                         break;
2751                 }
2752         }
2753
2754 skip_to_post_handle:
2755         if (ret != RTE_VHOST_MSG_RESULT_ERR &&
2756                         dev->extern_ops.post_msg_handle) {
2757                 ret = (*dev->extern_ops.post_msg_handle)(dev->vid,
2758                                 (void *)&msg);
2759                 switch (ret) {
2760                 case RTE_VHOST_MSG_RESULT_REPLY:
2761                         send_vhost_reply(fd, &msg);
2762                         /* Fall-through */
2763                 case RTE_VHOST_MSG_RESULT_ERR:
2764                 case RTE_VHOST_MSG_RESULT_OK:
2765                         handled = true;
2766                 case RTE_VHOST_MSG_RESULT_NOT_HANDLED:
2767                 default:
2768                         break;
2769                 }
2770         }
2771
2772         if (unlock_required)
2773                 vhost_user_unlock_all_queue_pairs(dev);
2774
2775         /* If message was not handled at this stage, treat it as an error */
2776         if (!handled) {
2777                 VHOST_LOG_CONFIG(ERR,
2778                         "vhost message (req: %d) was not handled.\n", request);
2779                 close_msg_fds(&msg);
2780                 ret = RTE_VHOST_MSG_RESULT_ERR;
2781         }
2782
2783         /*
2784          * If the request required a reply that was already sent,
2785          * this optional reply-ack won't be sent as the
2786          * VHOST_USER_NEED_REPLY was cleared in send_vhost_reply().
2787          */
2788         if (msg.flags & VHOST_USER_NEED_REPLY) {
2789                 msg.payload.u64 = ret == RTE_VHOST_MSG_RESULT_ERR;
2790                 msg.size = sizeof(msg.payload.u64);
2791                 msg.fd_num = 0;
2792                 send_vhost_reply(fd, &msg);
2793         } else if (ret == RTE_VHOST_MSG_RESULT_ERR) {
2794                 VHOST_LOG_CONFIG(ERR,
2795                         "vhost message handling failed.\n");
2796                 return -1;
2797         }
2798
2799         for (i = 0; i < dev->nr_vring; i++) {
2800                 struct vhost_virtqueue *vq = dev->virtqueue[i];
2801                 bool cur_ready = vq_is_ready(dev, vq);
2802
2803                 if (cur_ready != (vq && vq->ready)) {
2804                         vhost_user_notify_queue_state(dev, i, cur_ready);
2805                         vq->ready = cur_ready;
2806                 }
2807         }
2808
2809
2810         if (!(dev->flags & VIRTIO_DEV_RUNNING) && virtio_is_ready(dev)) {
2811                 dev->flags |= VIRTIO_DEV_READY;
2812
2813                 if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
2814                         if (dev->dequeue_zero_copy) {
2815                                 VHOST_LOG_CONFIG(INFO,
2816                                                 "dequeue zero copy is enabled\n");
2817                         }
2818
2819                         if (dev->notify_ops->new_device(dev->vid) == 0)
2820                                 dev->flags |= VIRTIO_DEV_RUNNING;
2821                 }
2822         }
2823
2824         vdpa_dev = dev->vdpa_dev;
2825         if (vdpa_dev && virtio_is_ready(dev) &&
2826             !(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
2827                 if (vdpa_dev->ops->dev_conf)
2828                         vdpa_dev->ops->dev_conf(dev->vid);
2829                 dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED;
2830         }
2831
2832         return 0;
2833 }
2834
2835 static int process_slave_message_reply(struct virtio_net *dev,
2836                                        const struct VhostUserMsg *msg)
2837 {
2838         struct VhostUserMsg msg_reply;
2839         int ret;
2840
2841         if ((msg->flags & VHOST_USER_NEED_REPLY) == 0)
2842                 return 0;
2843
2844         ret = read_vhost_message(dev->slave_req_fd, &msg_reply);
2845         if (ret <= 0) {
2846                 if (ret < 0)
2847                         VHOST_LOG_CONFIG(ERR,
2848                                 "vhost read slave message reply failed\n");
2849                 else
2850                         VHOST_LOG_CONFIG(INFO,
2851                                 "vhost peer closed\n");
2852                 ret = -1;
2853                 goto out;
2854         }
2855
2856         ret = 0;
2857         if (msg_reply.request.slave != msg->request.slave) {
2858                 VHOST_LOG_CONFIG(ERR,
2859                         "Received unexpected msg type (%u), expected %u\n",
2860                         msg_reply.request.slave, msg->request.slave);
2861                 ret = -1;
2862                 goto out;
2863         }
2864
2865         ret = msg_reply.payload.u64 ? -1 : 0;
2866
2867 out:
2868         rte_spinlock_unlock(&dev->slave_req_lock);
2869         return ret;
2870 }
2871
2872 int
2873 vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
2874 {
2875         int ret;
2876         struct VhostUserMsg msg = {
2877                 .request.slave = VHOST_USER_SLAVE_IOTLB_MSG,
2878                 .flags = VHOST_USER_VERSION,
2879                 .size = sizeof(msg.payload.iotlb),
2880                 .payload.iotlb = {
2881                         .iova = iova,
2882                         .perm = perm,
2883                         .type = VHOST_IOTLB_MISS,
2884                 },
2885         };
2886
2887         ret = send_vhost_message(dev->slave_req_fd, &msg);
2888         if (ret < 0) {
2889                 VHOST_LOG_CONFIG(ERR,
2890                                 "Failed to send IOTLB miss message (%d)\n",
2891                                 ret);
2892                 return ret;
2893         }
2894
2895         return 0;
2896 }
2897
2898 static int
2899 vhost_user_slave_config_change(struct virtio_net *dev, bool need_reply)
2900 {
2901         int ret;
2902         struct VhostUserMsg msg = {
2903                 .request.slave = VHOST_USER_SLAVE_CONFIG_CHANGE_MSG,
2904                 .flags = VHOST_USER_VERSION,
2905                 .size = 0,
2906         };
2907
2908         if (need_reply)
2909                 msg.flags |= VHOST_USER_NEED_REPLY;
2910
2911         ret = send_vhost_slave_message(dev, &msg);
2912         if (ret < 0) {
2913                 VHOST_LOG_CONFIG(ERR,
2914                                 "Failed to send config change (%d)\n",
2915                                 ret);
2916                 return ret;
2917         }
2918
2919         return process_slave_message_reply(dev, &msg);
2920 }
2921
2922 int
2923 rte_vhost_slave_config_change(int vid, bool need_reply)
2924 {
2925         struct virtio_net *dev;
2926
2927         dev = get_device(vid);
2928         if (!dev)
2929                 return -ENODEV;
2930
2931         return vhost_user_slave_config_change(dev, need_reply);
2932 }
2933
2934 static int vhost_user_slave_set_vring_host_notifier(struct virtio_net *dev,
2935                                                     int index, int fd,
2936                                                     uint64_t offset,
2937                                                     uint64_t size)
2938 {
2939         int ret;
2940         struct VhostUserMsg msg = {
2941                 .request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
2942                 .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY,
2943                 .size = sizeof(msg.payload.area),
2944                 .payload.area = {
2945                         .u64 = index & VHOST_USER_VRING_IDX_MASK,
2946                         .size = size,
2947                         .offset = offset,
2948                 },
2949         };
2950
2951         if (fd < 0)
2952                 msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
2953         else {
2954                 msg.fds[0] = fd;
2955                 msg.fd_num = 1;
2956         }
2957
2958         ret = send_vhost_slave_message(dev, &msg);
2959         if (ret < 0) {
2960                 VHOST_LOG_CONFIG(ERR,
2961                         "Failed to set host notifier (%d)\n", ret);
2962                 return ret;
2963         }
2964
2965         return process_slave_message_reply(dev, &msg);
2966 }
2967
2968 int rte_vhost_host_notifier_ctrl(int vid, uint16_t qid, bool enable)
2969 {
2970         struct virtio_net *dev;
2971         struct rte_vdpa_device *vdpa_dev;
2972         int vfio_device_fd, ret = 0;
2973         uint64_t offset, size;
2974         unsigned int i, q_start, q_last;
2975
2976         dev = get_device(vid);
2977         if (!dev)
2978                 return -ENODEV;
2979
2980         vdpa_dev = dev->vdpa_dev;
2981         if (vdpa_dev == NULL)
2982                 return -ENODEV;
2983
2984         if (!(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
2985             !(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) ||
2986             !(dev->protocol_features &
2987                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) ||
2988             !(dev->protocol_features &
2989                         (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD)) ||
2990             !(dev->protocol_features &
2991                         (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER)))
2992                 return -ENOTSUP;
2993
2994         if (qid == RTE_VHOST_QUEUE_ALL) {
2995                 q_start = 0;
2996                 q_last = dev->nr_vring - 1;
2997         } else {
2998                 if (qid >= dev->nr_vring)
2999                         return -EINVAL;
3000                 q_start = qid;
3001                 q_last = qid;
3002         }
3003
3004         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_vfio_device_fd, -ENOTSUP);
3005         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_notify_area, -ENOTSUP);
3006
3007         vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid);
3008         if (vfio_device_fd < 0)
3009                 return -ENOTSUP;
3010
3011         if (enable) {
3012                 for (i = q_start; i <= q_last; i++) {
3013                         if (vdpa_dev->ops->get_notify_area(vid, i, &offset,
3014                                         &size) < 0) {
3015                                 ret = -ENOTSUP;
3016                                 goto disable;
3017                         }
3018
3019                         if (vhost_user_slave_set_vring_host_notifier(dev, i,
3020                                         vfio_device_fd, offset, size) < 0) {
3021                                 ret = -EFAULT;
3022                                 goto disable;
3023                         }
3024                 }
3025         } else {
3026 disable:
3027                 for (i = q_start; i <= q_last; i++) {
3028                         vhost_user_slave_set_vring_host_notifier(dev, i, -1,
3029                                         0, 0);
3030                 }
3031         }
3032
3033         return ret;
3034 }