lib/librte_vhost/vhost_user.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39 #include <sys/mman.h>
  40 #include <sys/types.h>
  41 #include <sys/stat.h>
  42 #include <assert.h>
  43 #ifdef RTE_LIBRTE_VHOST_NUMA
  44 #include <numaif.h>
  45 #endif
  46
  47 #include <rte_common.h>
  48 #include <rte_malloc.h>
  49 #include <rte_log.h>
  50
  51 #include "vhost.h"
  52 #include "vhost_user.h"
  53
  54 static const char *vhost_message_str[VHOST_USER_MAX] = {
  55         [VHOST_USER_NONE] = "VHOST_USER_NONE",
  56         [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
  57         [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
  58         [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
  59         [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
  60         [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
  61         [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
  62         [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
  63         [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
  64         [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
  65         [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
  66         [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
  67         [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
  68         [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
  69         [VHOST_USER_SET_VRING_ERR]  = "VHOST_USER_SET_VRING_ERR",
  70         [VHOST_USER_GET_PROTOCOL_FEATURES]  = "VHOST_USER_GET_PROTOCOL_FEATURES",
  71         [VHOST_USER_SET_PROTOCOL_FEATURES]  = "VHOST_USER_SET_PROTOCOL_FEATURES",
  72         [VHOST_USER_GET_QUEUE_NUM]  = "VHOST_USER_GET_QUEUE_NUM",
  73         [VHOST_USER_SET_VRING_ENABLE]  = "VHOST_USER_SET_VRING_ENABLE",
  74         [VHOST_USER_SEND_RARP]  = "VHOST_USER_SEND_RARP",
  75 };
  76
  77 struct orig_region_map {
  78         int fd;
  79         uint64_t mapped_address;
  80         uint64_t mapped_size;
  81         uint64_t blksz;
  82 };
  83
  84 #define orig_region(ptr, nregions) \
  85         ((struct orig_region_map *)RTE_PTR_ADD((ptr), \
  86                 sizeof(struct virtio_memory) + \
  87                 sizeof(struct virtio_memory_regions) * (nregions)))
  88
  89 static uint64_t
  90 get_blk_size(int fd)
  91 {
  92         struct stat stat;
  93         int ret;
  94
  95         ret = fstat(fd, &stat);
  96         return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
  97 }
  98
  99 static void
 100 free_mem_region(struct virtio_net *dev)
 101 {
 102         struct orig_region_map *region;
 103         unsigned int idx;
 104
 105         if (!dev || !dev->mem)
 106                 return;
 107
 108         region = orig_region(dev->mem, dev->mem->nregions);
 109         for (idx = 0; idx < dev->mem->nregions; idx++) {
 110                 if (region[idx].mapped_address) {
 111                         munmap((void *)(uintptr_t)region[idx].mapped_address,
 112                                         region[idx].mapped_size);
 113                         close(region[idx].fd);
 114                 }
 115         }
 116 }
 117
 118 void
 119 vhost_backend_cleanup(struct virtio_net *dev)
 120 {
 121         if (dev->mem) {
 122                 free_mem_region(dev);
 123                 free(dev->mem);
 124                 dev->mem = NULL;
 125         }
 126         if (dev->log_addr) {
 127                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 128                 dev->log_addr = 0;
 129         }
 130 }
 131
 132 /*
 133  * This function just returns success at the moment unless
 134  * the device hasn't been initialised.
 135  */
 136 static int
 137 vhost_set_owner(int vid)
 138 {
 139         struct virtio_net *dev;
 140
 141         dev = get_device(vid);
 142         if (dev == NULL)
 143                 return -1;
 144
 145         return 0;
 146 }
 147
 148 static int
 149 vhost_reset_owner(int vid)
 150 {
 151         struct virtio_net *dev;
 152
 153         dev = get_device(vid);
 154         if (dev == NULL)
 155                 return -1;
 156
 157         if (dev->flags & VIRTIO_DEV_RUNNING) {
 158                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 159                 notify_ops->destroy_device(vid);
 160         }
 161
 162         cleanup_device(dev, 0);
 163         reset_device(dev);
 164         return 0;
 165 }
 166
 167 /*
 168  * The features that we support are requested.
 169  */
 170 static int
 171 vhost_get_features(int vid, uint64_t *pu)
 172 {
 173         struct virtio_net *dev;
 174
 175         dev = get_device(vid);
 176         if (dev == NULL)
 177                 return -1;
 178
 179         /* Send our supported features. */
 180         *pu = VHOST_FEATURES;
 181         return 0;
 182 }
 183
 184 /*
 185  * We receive the negotiated features supported by us and the virtio device.
 186  */
 187 static int
 188 vhost_set_features(int vid, uint64_t *pu)
 189 {
 190         struct virtio_net *dev;
 191
 192         dev = get_device(vid);
 193         if (dev == NULL)
 194                 return -1;
 195         if (*pu & ~VHOST_FEATURES)
 196                 return -1;
 197
 198         dev->features = *pu;
 199         if (dev->features &
 200                 ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
 201                 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 202         } else {
 203                 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
 204         }
 205         LOG_DEBUG(VHOST_CONFIG,
 206                 "(%d) mergeable RX buffers %s, virtio 1 %s\n",
 207                 dev->vid,
 208                 (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
 209                 (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
 210
 211         return 0;
 212 }
 213
 214 /*
 215  * The virtio device sends us the size of the descriptor ring.
 216  */
 217 static int
 218 vhost_set_vring_num(int vid, struct vhost_vring_state *state)
 219 {
 220         struct virtio_net *dev;
 221
 222         dev = get_device(vid);
 223         if (dev == NULL)
 224                 return -1;
 225
 226         /* State->index refers to the queue index. The txq is 1, rxq is 0. */
 227         dev->virtqueue[state->index]->size = state->num;
 228
 229         return 0;
 230 }
 231
 232 /*
 233  * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the
 234  * same numa node as the memory of vring descriptor.
 235  */
 236 #ifdef RTE_LIBRTE_VHOST_NUMA
 237 static struct virtio_net*
 238 numa_realloc(struct virtio_net *dev, int index)
 239 {
 240         int oldnode, newnode;
 241         struct virtio_net *old_dev;
 242         struct vhost_virtqueue *old_vq, *vq;
 243         int ret;
 244
 245         /*
 246          * vq is allocated on pairs, we should try to do realloc
 247          * on first queue of one queue pair only.
 248          */
 249         if (index % VIRTIO_QNUM != 0)
 250                 return dev;
 251
 252         old_dev = dev;
 253         vq = old_vq = dev->virtqueue[index];
 254
 255         ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc,
 256                             MPOL_F_NODE | MPOL_F_ADDR);
 257
 258         /* check if we need to reallocate vq */
 259         ret |= get_mempolicy(&oldnode, NULL, 0, old_vq,
 260                              MPOL_F_NODE | MPOL_F_ADDR);
 261         if (ret) {
 262                 RTE_LOG(ERR, VHOST_CONFIG,
 263                         "Unable to get vq numa information.\n");
 264                 return dev;
 265         }
 266         if (oldnode != newnode) {
 267                 RTE_LOG(INFO, VHOST_CONFIG,
 268                         "reallocate vq from %d to %d node\n", oldnode, newnode);
 269                 vq = rte_malloc_socket(NULL, sizeof(*vq) * VIRTIO_QNUM, 0,
 270                                        newnode);
 271                 if (!vq)
 272                         return dev;
 273
 274                 memcpy(vq, old_vq, sizeof(*vq) * VIRTIO_QNUM);
 275                 rte_free(old_vq);
 276         }
 277
 278         /* check if we need to reallocate dev */
 279         ret = get_mempolicy(&oldnode, NULL, 0, old_dev,
 280                             MPOL_F_NODE | MPOL_F_ADDR);
 281         if (ret) {
 282                 RTE_LOG(ERR, VHOST_CONFIG,
 283                         "Unable to get dev numa information.\n");
 284                 goto out;
 285         }
 286         if (oldnode != newnode) {
 287                 RTE_LOG(INFO, VHOST_CONFIG,
 288                         "reallocate dev from %d to %d node\n",
 289                         oldnode, newnode);
 290                 dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode);
 291                 if (!dev) {
 292                         dev = old_dev;
 293                         goto out;
 294                 }
 295
 296                 memcpy(dev, old_dev, sizeof(*dev));
 297                 rte_free(old_dev);
 298         }
 299
 300 out:
 301         dev->virtqueue[index] = vq;
 302         dev->virtqueue[index + 1] = vq + 1;
 303         vhost_devices[dev->vid] = dev;
 304
 305         return dev;
 306 }
 307 #else
 308 static struct virtio_net*
 309 numa_realloc(struct virtio_net *dev, int index __rte_unused)
 310 {
 311         return dev;
 312 }
 313 #endif
 314
 315 /*
 316  * Converts QEMU virtual address to Vhost virtual address. This function is
 317  * used to convert the ring addresses to our address space.
 318  */
 319 static uint64_t
 320 qva_to_vva(struct virtio_net *dev, uint64_t qemu_va)
 321 {
 322         struct virtio_memory_regions *region;
 323         uint64_t vhost_va = 0;
 324         uint32_t regionidx = 0;
 325
 326         /* Find the region where the address lives. */
 327         for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
 328                 region = &dev->mem->regions[regionidx];
 329                 if ((qemu_va >= region->userspace_address) &&
 330                         (qemu_va <= region->userspace_address +
 331                         region->memory_size)) {
 332                         vhost_va = qemu_va + region->guest_phys_address +
 333                                 region->address_offset -
 334                                 region->userspace_address;
 335                         break;
 336                 }
 337         }
 338         return vhost_va;
 339 }
 340
 341 /*
 342  * The virtio device sends us the desc, used and avail ring addresses.
 343  * This function then converts these to our address space.
 344  */
 345 static int
 346 vhost_set_vring_addr(int vid, struct vhost_vring_addr *addr)
 347 {
 348         struct virtio_net *dev;
 349         struct vhost_virtqueue *vq;
 350
 351         dev = get_device(vid);
 352         if ((dev == NULL) || (dev->mem == NULL))
 353                 return -1;
 354
 355         /* addr->index refers to the queue index. The txq 1, rxq is 0. */
 356         vq = dev->virtqueue[addr->index];
 357
 358         /* The addresses are converted from QEMU virtual to Vhost virtual. */
 359         vq->desc = (struct vring_desc *)(uintptr_t)qva_to_vva(dev,
 360                         addr->desc_user_addr);
 361         if (vq->desc == 0) {
 362                 RTE_LOG(ERR, VHOST_CONFIG,
 363                         "(%d) failed to find desc ring address.\n",
 364                         dev->vid);
 365                 return -1;
 366         }
 367
 368         dev = numa_realloc(dev, addr->index);
 369         vq = dev->virtqueue[addr->index];
 370
 371         vq->avail = (struct vring_avail *)(uintptr_t)qva_to_vva(dev,
 372                         addr->avail_user_addr);
 373         if (vq->avail == 0) {
 374                 RTE_LOG(ERR, VHOST_CONFIG,
 375                         "(%d) failed to find avail ring address.\n",
 376                         dev->vid);
 377                 return -1;
 378         }
 379
 380         vq->used = (struct vring_used *)(uintptr_t)qva_to_vva(dev,
 381                         addr->used_user_addr);
 382         if (vq->used == 0) {
 383                 RTE_LOG(ERR, VHOST_CONFIG,
 384                         "(%d) failed to find used ring address.\n",
 385                         dev->vid);
 386                 return -1;
 387         }
 388
 389         if (vq->last_used_idx != vq->used->idx) {
 390                 RTE_LOG(WARNING, VHOST_CONFIG,
 391                         "last_used_idx (%u) and vq->used->idx (%u) mismatches; "
 392                         "some packets maybe resent for Tx and dropped for Rx\n",
 393                         vq->last_used_idx, vq->used->idx);
 394                 vq->last_used_idx     = vq->used->idx;
 395         }
 396
 397         vq->log_guest_addr = addr->log_guest_addr;
 398
 399         LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
 400                         dev->vid, vq->desc);
 401         LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n",
 402                         dev->vid, vq->avail);
 403         LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n",
 404                         dev->vid, vq->used);
 405         LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n",
 406                         dev->vid, vq->log_guest_addr);
 407
 408         return 0;
 409 }
 410
 411 /*
 412  * The virtio device sends us the available ring last used index.
 413  */
 414 static int
 415 vhost_set_vring_base(int vid, struct vhost_vring_state *state)
 416 {
 417         struct virtio_net *dev;
 418
 419         dev = get_device(vid);
 420         if (dev == NULL)
 421                 return -1;
 422
 423         /* State->index refers to the queue index. The txq is 1, rxq is 0. */
 424         dev->virtqueue[state->index]->last_used_idx = state->num;
 425
 426         return 0;
 427 }
 428
 429 /*
 430  * We send the virtio device our available ring last used index.
 431  */
 432 static int
 433 vhost_get_vring_base(int vid, uint32_t index,
 434         struct vhost_vring_state *state)
 435 {
 436         struct virtio_net *dev;
 437
 438         dev = get_device(vid);
 439         if (dev == NULL)
 440                 return -1;
 441
 442         state->index = index;
 443         /* State->index refers to the queue index. The txq is 1, rxq is 0. */
 444         state->num = dev->virtqueue[state->index]->last_used_idx;
 445
 446         return 0;
 447 }
 448
 449 /*
 450  * The virtio device sends an eventfd to interrupt the guest. This fd gets
 451  * copied into our process space.
 452  */
 453 static int
 454 vhost_set_vring_call(int vid, struct vhost_vring_file *file)
 455 {
 456         struct virtio_net *dev;
 457         struct vhost_virtqueue *vq;
 458         uint32_t cur_qp_idx = file->index / VIRTIO_QNUM;
 459
 460         dev = get_device(vid);
 461         if (dev == NULL)
 462                 return -1;
 463
 464         /*
 465          * FIXME: VHOST_SET_VRING_CALL is the first per-vring message
 466          * we get, so we do vring queue pair allocation here.
 467          */
 468         if (cur_qp_idx + 1 > dev->virt_qp_nb) {
 469                 if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0)
 470                         return -1;
 471         }
 472
 473         /* file->index refers to the queue index. The txq is 1, rxq is 0. */
 474         vq = dev->virtqueue[file->index];
 475         assert(vq != NULL);
 476
 477         if (vq->callfd >= 0)
 478                 close(vq->callfd);
 479
 480         vq->callfd = file->fd;
 481
 482         return 0;
 483 }
 484
 485 /*
 486  * The virtio device sends an eventfd that it can use to notify us.
 487  * This fd gets copied into our process space.
 488  */
 489 static int
 490 vhost_set_vring_kick(int vid, struct vhost_vring_file *file)
 491 {
 492         struct virtio_net *dev;
 493         struct vhost_virtqueue *vq;
 494
 495         dev = get_device(vid);
 496         if (dev == NULL)
 497                 return -1;
 498
 499         /* file->index refers to the queue index. The txq is 1, rxq is 0. */
 500         vq = dev->virtqueue[file->index];
 501
 502         if (vq->kickfd >= 0)
 503                 close(vq->kickfd);
 504
 505         vq->kickfd = file->fd;
 506
 507         return 0;
 508 }
 509
 510 static int
 511 user_set_mem_table(int vid, struct VhostUserMsg *pmsg)
 512 {
 513         struct VhostUserMemory memory = pmsg->payload.memory;
 514         struct virtio_memory_regions *pregion;
 515         uint64_t mapped_address, mapped_size;
 516         struct virtio_net *dev;
 517         unsigned int idx = 0;
 518         struct orig_region_map *pregion_orig;
 519         uint64_t alignment;
 520
 521         /* unmap old memory regions one by one*/
 522         dev = get_device(vid);
 523         if (dev == NULL)
 524                 return -1;
 525
 526         /* Remove from the data plane. */
 527         if (dev->flags & VIRTIO_DEV_RUNNING) {
 528                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 529                 notify_ops->destroy_device(vid);
 530         }
 531
 532         if (dev->mem) {
 533                 free_mem_region(dev);
 534                 free(dev->mem);
 535                 dev->mem = NULL;
 536         }
 537
 538         dev->mem = calloc(1,
 539                 sizeof(struct virtio_memory) +
 540                 sizeof(struct virtio_memory_regions) * memory.nregions +
 541                 sizeof(struct orig_region_map) * memory.nregions);
 542         if (dev->mem == NULL) {
 543                 RTE_LOG(ERR, VHOST_CONFIG,
 544                         "(%d) failed to allocate memory for dev->mem\n",
 545                         dev->vid);
 546                 return -1;
 547         }
 548         dev->mem->nregions = memory.nregions;
 549
 550         pregion_orig = orig_region(dev->mem, memory.nregions);
 551         for (idx = 0; idx < memory.nregions; idx++) {
 552                 pregion = &dev->mem->regions[idx];
 553                 pregion->guest_phys_address =
 554                         memory.regions[idx].guest_phys_addr;
 555                 pregion->guest_phys_address_end =
 556                         memory.regions[idx].guest_phys_addr +
 557                         memory.regions[idx].memory_size;
 558                 pregion->memory_size =
 559                         memory.regions[idx].memory_size;
 560                 pregion->userspace_address =
 561                         memory.regions[idx].userspace_addr;
 562
 563                 /* This is ugly */
 564                 mapped_size = memory.regions[idx].memory_size +
 565                         memory.regions[idx].mmap_offset;
 566
 567                 /* mmap() without flag of MAP_ANONYMOUS, should be called
 568                  * with length argument aligned with hugepagesz at older
 569                  * longterm version Linux, like 2.6.32 and 3.2.72, or
 570                  * mmap() will fail with EINVAL.
 571                  *
 572                  * to avoid failure, make sure in caller to keep length
 573                  * aligned.
 574                  */
 575                 alignment = get_blk_size(pmsg->fds[idx]);
 576                 if (alignment == (uint64_t)-1) {
 577                         RTE_LOG(ERR, VHOST_CONFIG,
 578                                 "couldn't get hugepage size through fstat\n");
 579                         goto err_mmap;
 580                 }
 581                 mapped_size = RTE_ALIGN_CEIL(mapped_size, alignment);
 582
 583                 mapped_address = (uint64_t)(uintptr_t)mmap(NULL,
 584                         mapped_size,
 585                         PROT_READ | PROT_WRITE, MAP_SHARED,
 586                         pmsg->fds[idx],
 587                         0);
 588
 589                 RTE_LOG(INFO, VHOST_CONFIG,
 590                         "mapped region %d fd:%d to:%p sz:0x%"PRIx64" "
 591                         "off:0x%"PRIx64" align:0x%"PRIx64"\n",
 592                         idx, pmsg->fds[idx], (void *)(uintptr_t)mapped_address,
 593                         mapped_size, memory.regions[idx].mmap_offset,
 594                         alignment);
 595
 596                 if (mapped_address == (uint64_t)(uintptr_t)MAP_FAILED) {
 597                         RTE_LOG(ERR, VHOST_CONFIG,
 598                                 "mmap qemu guest failed.\n");
 599                         goto err_mmap;
 600                 }
 601
 602                 pregion_orig[idx].mapped_address = mapped_address;
 603                 pregion_orig[idx].mapped_size = mapped_size;
 604                 pregion_orig[idx].blksz = alignment;
 605                 pregion_orig[idx].fd = pmsg->fds[idx];
 606
 607                 mapped_address +=  memory.regions[idx].mmap_offset;
 608
 609                 pregion->address_offset = mapped_address -
 610                         pregion->guest_phys_address;
 611
 612                 if (memory.regions[idx].guest_phys_addr == 0) {
 613                         dev->mem->base_address =
 614                                 memory.regions[idx].userspace_addr;
 615                         dev->mem->mapped_address =
 616                                 pregion->address_offset;
 617                 }
 618
 619                 LOG_DEBUG(VHOST_CONFIG,
 620                         "REGION: %u GPA: %p QEMU VA: %p SIZE (%"PRIu64")\n",
 621                         idx,
 622                         (void *)(uintptr_t)pregion->guest_phys_address,
 623                         (void *)(uintptr_t)pregion->userspace_address,
 624                          pregion->memory_size);
 625         }
 626
 627         return 0;
 628
 629 err_mmap:
 630         while (idx--) {
 631                 munmap((void *)(uintptr_t)pregion_orig[idx].mapped_address,
 632                                 pregion_orig[idx].mapped_size);
 633                 close(pregion_orig[idx].fd);
 634         }
 635         free(dev->mem);
 636         dev->mem = NULL;
 637         return -1;
 638 }
 639
 640 static int
 641 vq_is_ready(struct vhost_virtqueue *vq)
 642 {
 643         return vq && vq->desc   &&
 644                vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
 645                vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD;
 646 }
 647
 648 static int
 649 virtio_is_ready(struct virtio_net *dev)
 650 {
 651         struct vhost_virtqueue *rvq, *tvq;
 652         uint32_t i;
 653
 654         for (i = 0; i < dev->virt_qp_nb; i++) {
 655                 rvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ];
 656                 tvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ];
 657
 658                 if (!vq_is_ready(rvq) || !vq_is_ready(tvq)) {
 659                         RTE_LOG(INFO, VHOST_CONFIG,
 660                                 "virtio is not ready for processing.\n");
 661                         return 0;
 662                 }
 663         }
 664
 665         RTE_LOG(INFO, VHOST_CONFIG,
 666                 "virtio is now ready for processing.\n");
 667         return 1;
 668 }
 669
 670 static void
 671 user_set_vring_call(int vid, struct VhostUserMsg *pmsg)
 672 {
 673         struct vhost_vring_file file;
 674
 675         file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
 676         if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
 677                 file.fd = VIRTIO_INVALID_EVENTFD;
 678         else
 679                 file.fd = pmsg->fds[0];
 680         RTE_LOG(INFO, VHOST_CONFIG,
 681                 "vring call idx:%d file:%d\n", file.index, file.fd);
 682         vhost_set_vring_call(vid, &file);
 683 }
 684
 685 /*
 686  *  In vhost-user, when we receive kick message, will test whether virtio
 687  *  device is ready for packet processing.
 688  */
 689 static void
 690 user_set_vring_kick(int vid, struct VhostUserMsg *pmsg)
 691 {
 692         struct vhost_vring_file file;
 693         struct virtio_net *dev = get_device(vid);
 694
 695         if (!dev)
 696                 return;
 697
 698         file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
 699         if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
 700                 file.fd = VIRTIO_INVALID_EVENTFD;
 701         else
 702                 file.fd = pmsg->fds[0];
 703         RTE_LOG(INFO, VHOST_CONFIG,
 704                 "vring kick idx:%d file:%d\n", file.index, file.fd);
 705         vhost_set_vring_kick(vid, &file);
 706
 707         if (virtio_is_ready(dev) && !(dev->flags & VIRTIO_DEV_RUNNING)) {
 708                 if (notify_ops->new_device(vid) == 0)
 709                         dev->flags |= VIRTIO_DEV_RUNNING;
 710         }
 711 }
 712
 713 /*
 714  * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
 715  */
 716 static int
 717 user_get_vring_base(int vid, struct vhost_vring_state *state)
 718 {
 719         struct virtio_net *dev = get_device(vid);
 720
 721         if (dev == NULL)
 722                 return -1;
 723         /* We have to stop the queue (virtio) if it is running. */
 724         if (dev->flags & VIRTIO_DEV_RUNNING) {
 725                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 726                 notify_ops->destroy_device(vid);
 727         }
 728
 729         /* Here we are safe to get the last used index */
 730         vhost_get_vring_base(vid, state->index, state);
 731
 732         RTE_LOG(INFO, VHOST_CONFIG,
 733                 "vring base idx:%d file:%d\n", state->index, state->num);
 734         /*
 735          * Based on current qemu vhost-user implementation, this message is
 736          * sent and only sent in vhost_vring_stop.
 737          * TODO: cleanup the vring, it isn't usable since here.
 738          */
 739         if (dev->virtqueue[state->index]->kickfd >= 0)
 740                 close(dev->virtqueue[state->index]->kickfd);
 741
 742         dev->virtqueue[state->index]->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
 743
 744         return 0;
 745 }
 746
 747 /*
 748  * when virtio queues are ready to work, qemu will send us to
 749  * enable the virtio queue pair.
 750  */
 751 static int
 752 user_set_vring_enable(int vid, struct vhost_vring_state *state)
 753 {
 754         struct virtio_net *dev;
 755         int enable = (int)state->num;
 756
 757         dev = get_device(vid);
 758         if (dev == NULL)
 759                 return -1;
 760
 761         RTE_LOG(INFO, VHOST_CONFIG,
 762                 "set queue enable: %d to qp idx: %d\n",
 763                 enable, state->index);
 764
 765         if (notify_ops->vring_state_changed)
 766                 notify_ops->vring_state_changed(vid, state->index, enable);
 767
 768         dev->virtqueue[state->index]->enabled = enable;
 769
 770         return 0;
 771 }
 772
 773 static void
 774 user_set_protocol_features(int vid, uint64_t protocol_features)
 775 {
 776         struct virtio_net *dev;
 777
 778         dev = get_device(vid);
 779         if (dev == NULL || protocol_features & ~VHOST_USER_PROTOCOL_FEATURES)
 780                 return;
 781
 782         dev->protocol_features = protocol_features;
 783 }
 784
 785 static int
 786 user_set_log_base(int vid, struct VhostUserMsg *msg)
 787 {
 788         struct virtio_net *dev;
 789         int fd = msg->fds[0];
 790         uint64_t size, off;
 791         void *addr;
 792
 793         dev = get_device(vid);
 794         if (!dev)
 795                 return -1;
 796
 797         if (fd < 0) {
 798                 RTE_LOG(ERR, VHOST_CONFIG, "invalid log fd: %d\n", fd);
 799                 return -1;
 800         }
 801
 802         if (msg->size != sizeof(VhostUserLog)) {
 803                 RTE_LOG(ERR, VHOST_CONFIG,
 804                         "invalid log base msg size: %"PRId32" != %d\n",
 805                         msg->size, (int)sizeof(VhostUserLog));
 806                 return -1;
 807         }
 808
 809         size = msg->payload.log.mmap_size;
 810         off  = msg->payload.log.mmap_offset;
 811         RTE_LOG(INFO, VHOST_CONFIG,
 812                 "log mmap size: %"PRId64", offset: %"PRId64"\n",
 813                 size, off);
 814
 815         /*
 816          * mmap from 0 to workaround a hugepage mmap bug: mmap will
 817          * fail when offset is not page size aligned.
 818          */
 819         addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
 820         close(fd);
 821         if (addr == MAP_FAILED) {
 822                 RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n");
 823                 return -1;
 824         }
 825
 826         /*
 827          * Free previously mapped log memory on occasionally
 828          * multiple VHOST_USER_SET_LOG_BASE.
 829          */
 830         if (dev->log_addr) {
 831                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 832         }
 833         dev->log_addr = (uint64_t)(uintptr_t)addr;
 834         dev->log_base = dev->log_addr + off;
 835         dev->log_size = size;
 836
 837         return 0;
 838 }
 839
 840 /*
 841  * An rarp packet is constructed and broadcasted to notify switches about
 842  * the new location of the migrated VM, so that packets from outside will
 843  * not be lost after migration.
 844  *
 845  * However, we don't actually "send" a rarp packet here, instead, we set
 846  * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
 847  */
 848 static int
 849 user_send_rarp(int vid, struct VhostUserMsg *msg)
 850 {
 851         struct virtio_net *dev;
 852         uint8_t *mac = (uint8_t *)&msg->payload.u64;
 853
 854         dev = get_device(vid);
 855         if (!dev)
 856                 return -1;
 857
 858         RTE_LOG(DEBUG, VHOST_CONFIG,
 859                 ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
 860                 mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
 861         memcpy(dev->mac.addr_bytes, mac, 6);
 862
 863         /*
 864          * Set the flag to inject a RARP broadcast packet at
 865          * rte_vhost_dequeue_burst().
 866          *
 867          * rte_smp_wmb() is for making sure the mac is copied
 868          * before the flag is set.
 869          */
 870         rte_smp_wmb();
 871         rte_atomic16_set(&dev->broadcast_rarp, 1);
 872
 873         return 0;
 874 }
 875
 876 /* return bytes# of read on success or negative val on failure. */
 877 static int
 878 read_vhost_message(int sockfd, struct VhostUserMsg *msg)
 879 {
 880         int ret;
 881
 882         ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
 883                 msg->fds, VHOST_MEMORY_MAX_NREGIONS);
 884         if (ret <= 0)
 885                 return ret;
 886
 887         if (msg && msg->size) {
 888                 if (msg->size > sizeof(msg->payload)) {
 889                         RTE_LOG(ERR, VHOST_CONFIG,
 890                                 "invalid msg size: %d\n", msg->size);
 891                         return -1;
 892                 }
 893                 ret = read(sockfd, &msg->payload, msg->size);
 894                 if (ret <= 0)
 895                         return ret;
 896                 if (ret != (int)msg->size) {
 897                         RTE_LOG(ERR, VHOST_CONFIG,
 898                                 "read control message failed\n");
 899                         return -1;
 900                 }
 901         }
 902
 903         return ret;
 904 }
 905
 906 static int
 907 send_vhost_message(int sockfd, struct VhostUserMsg *msg)
 908 {
 909         int ret;
 910
 911         if (!msg)
 912                 return 0;
 913
 914         msg->flags &= ~VHOST_USER_VERSION_MASK;
 915         msg->flags |= VHOST_USER_VERSION;
 916         msg->flags |= VHOST_USER_REPLY_MASK;
 917
 918         ret = send_fd_message(sockfd, (char *)msg,
 919                 VHOST_USER_HDR_SIZE + msg->size, NULL, 0);
 920
 921         return ret;
 922 }
 923
 924 int
 925 vhost_user_msg_handler(int vid, int fd)
 926 {
 927         struct VhostUserMsg msg;
 928         uint64_t features = 0;
 929         int ret;
 930
 931         ret = read_vhost_message(fd, &msg);
 932         if (ret <= 0 || msg.request >= VHOST_USER_MAX) {
 933                 if (ret < 0)
 934                         RTE_LOG(ERR, VHOST_CONFIG,
 935                                 "vhost read message failed\n");
 936                 else if (ret == 0)
 937                         RTE_LOG(INFO, VHOST_CONFIG,
 938                                 "vhost peer closed\n");
 939                 else
 940                         RTE_LOG(ERR, VHOST_CONFIG,
 941                                 "vhost read incorrect message\n");
 942
 943                 return -1;
 944         }
 945
 946         RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n",
 947                 vhost_message_str[msg.request]);
 948         switch (msg.request) {
 949         case VHOST_USER_GET_FEATURES:
 950                 ret = vhost_get_features(vid, &features);
 951                 msg.payload.u64 = features;
 952                 msg.size = sizeof(msg.payload.u64);
 953                 send_vhost_message(fd, &msg);
 954                 break;
 955         case VHOST_USER_SET_FEATURES:
 956                 features = msg.payload.u64;
 957                 vhost_set_features(vid, &features);
 958                 break;
 959
 960         case VHOST_USER_GET_PROTOCOL_FEATURES:
 961                 msg.payload.u64 = VHOST_USER_PROTOCOL_FEATURES;
 962                 msg.size = sizeof(msg.payload.u64);
 963                 send_vhost_message(fd, &msg);
 964                 break;
 965         case VHOST_USER_SET_PROTOCOL_FEATURES:
 966                 user_set_protocol_features(vid, msg.payload.u64);
 967                 break;
 968
 969         case VHOST_USER_SET_OWNER:
 970                 vhost_set_owner(vid);
 971                 break;
 972         case VHOST_USER_RESET_OWNER:
 973                 vhost_reset_owner(vid);
 974                 break;
 975
 976         case VHOST_USER_SET_MEM_TABLE:
 977                 user_set_mem_table(vid, &msg);
 978                 break;
 979
 980         case VHOST_USER_SET_LOG_BASE:
 981                 user_set_log_base(vid, &msg);
 982
 983                 /* it needs a reply */
 984                 msg.size = sizeof(msg.payload.u64);
 985                 send_vhost_message(fd, &msg);
 986                 break;
 987         case VHOST_USER_SET_LOG_FD:
 988                 close(msg.fds[0]);
 989                 RTE_LOG(INFO, VHOST_CONFIG, "not implemented.\n");
 990                 break;
 991
 992         case VHOST_USER_SET_VRING_NUM:
 993                 vhost_set_vring_num(vid, &msg.payload.state);
 994                 break;
 995         case VHOST_USER_SET_VRING_ADDR:
 996                 vhost_set_vring_addr(vid, &msg.payload.addr);
 997                 break;
 998         case VHOST_USER_SET_VRING_BASE:
 999                 vhost_set_vring_base(vid, &msg.payload.state);
1000                 break;
1001
1002         case VHOST_USER_GET_VRING_BASE:
1003                 ret = user_get_vring_base(vid, &msg.payload.state);
1004                 msg.size = sizeof(msg.payload.state);
1005                 send_vhost_message(fd, &msg);
1006                 break;
1007
1008         case VHOST_USER_SET_VRING_KICK:
1009                 user_set_vring_kick(vid, &msg);
1010                 break;
1011         case VHOST_USER_SET_VRING_CALL:
1012                 user_set_vring_call(vid, &msg);
1013                 break;
1014
1015         case VHOST_USER_SET_VRING_ERR:
1016                 if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1017                         close(msg.fds[0]);
1018                 RTE_LOG(INFO, VHOST_CONFIG, "not implemented\n");
1019                 break;
1020
1021         case VHOST_USER_GET_QUEUE_NUM:
1022                 msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS;
1023                 msg.size = sizeof(msg.payload.u64);
1024                 send_vhost_message(fd, &msg);
1025                 break;
1026
1027         case VHOST_USER_SET_VRING_ENABLE:
1028                 user_set_vring_enable(vid, &msg.payload.state);
1029                 break;
1030         case VHOST_USER_SEND_RARP:
1031                 user_send_rarp(vid, &msg);
1032                 break;
1033
1034         default:
1035                 break;
1036
1037         }
1038
1039         return 0;
1040 }