lib/librte_vhost/vhost_user.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39 #include <sys/mman.h>
  40 #include <sys/types.h>
  41 #include <sys/stat.h>
  42 #include <assert.h>
  43 #ifdef RTE_LIBRTE_VHOST_NUMA
  44 #include <numaif.h>
  45 #endif
  46
  47 #include <rte_common.h>
  48 #include <rte_malloc.h>
  49 #include <rte_log.h>
  50
  51 #include "vhost.h"
  52 #include "vhost_user.h"
  53
  54 static const char *vhost_message_str[VHOST_USER_MAX] = {
  55         [VHOST_USER_NONE] = "VHOST_USER_NONE",
  56         [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
  57         [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
  58         [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
  59         [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
  60         [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
  61         [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
  62         [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
  63         [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
  64         [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
  65         [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
  66         [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
  67         [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
  68         [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
  69         [VHOST_USER_SET_VRING_ERR]  = "VHOST_USER_SET_VRING_ERR",
  70         [VHOST_USER_GET_PROTOCOL_FEATURES]  = "VHOST_USER_GET_PROTOCOL_FEATURES",
  71         [VHOST_USER_SET_PROTOCOL_FEATURES]  = "VHOST_USER_SET_PROTOCOL_FEATURES",
  72         [VHOST_USER_GET_QUEUE_NUM]  = "VHOST_USER_GET_QUEUE_NUM",
  73         [VHOST_USER_SET_VRING_ENABLE]  = "VHOST_USER_SET_VRING_ENABLE",
  74         [VHOST_USER_SEND_RARP]  = "VHOST_USER_SEND_RARP",
  75 };
  76
  77 struct orig_region_map {
  78         int fd;
  79         uint64_t mapped_address;
  80         uint64_t mapped_size;
  81         uint64_t blksz;
  82 };
  83
  84 #define orig_region(ptr, nregions) \
  85         ((struct orig_region_map *)RTE_PTR_ADD((ptr), \
  86                 sizeof(struct virtio_memory) + \
  87                 sizeof(struct virtio_memory_regions) * (nregions)))
  88
  89 static uint64_t
  90 get_blk_size(int fd)
  91 {
  92         struct stat stat;
  93         int ret;
  94
  95         ret = fstat(fd, &stat);
  96         return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
  97 }
  98
  99 static void
 100 free_mem_region(struct virtio_net *dev)
 101 {
 102         struct orig_region_map *region;
 103         unsigned int idx;
 104
 105         if (!dev || !dev->mem)
 106                 return;
 107
 108         region = orig_region(dev->mem, dev->mem->nregions);
 109         for (idx = 0; idx < dev->mem->nregions; idx++) {
 110                 if (region[idx].mapped_address) {
 111                         munmap((void *)(uintptr_t)region[idx].mapped_address,
 112                                         region[idx].mapped_size);
 113                         close(region[idx].fd);
 114                 }
 115         }
 116 }
 117
 118 void
 119 vhost_backend_cleanup(struct virtio_net *dev)
 120 {
 121         if (dev->mem) {
 122                 free_mem_region(dev);
 123                 free(dev->mem);
 124                 dev->mem = NULL;
 125         }
 126         if (dev->log_addr) {
 127                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 128                 dev->log_addr = 0;
 129         }
 130 }
 131
 132 /*
 133  * This function just returns success at the moment unless
 134  * the device hasn't been initialised.
 135  */
 136 static int
 137 vhost_set_owner(int vid)
 138 {
 139         struct virtio_net *dev;
 140
 141         dev = get_device(vid);
 142         if (dev == NULL)
 143                 return -1;
 144
 145         return 0;
 146 }
 147
 148 static int
 149 vhost_reset_owner(int vid)
 150 {
 151         struct virtio_net *dev;
 152
 153         dev = get_device(vid);
 154         if (dev == NULL)
 155                 return -1;
 156
 157         if (dev->flags & VIRTIO_DEV_RUNNING) {
 158                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 159                 notify_ops->destroy_device(vid);
 160         }
 161
 162         cleanup_device(dev, 0);
 163         reset_device(dev);
 164         return 0;
 165 }
 166
 167 /*
 168  * The features that we support are requested.
 169  */
 170 static int
 171 vhost_get_features(int vid, uint64_t *pu)
 172 {
 173         struct virtio_net *dev;
 174
 175         dev = get_device(vid);
 176         if (dev == NULL)
 177                 return -1;
 178
 179         /* Send our supported features. */
 180         *pu = VHOST_FEATURES;
 181         return 0;
 182 }
 183
 184 /*
 185  * We receive the negotiated features supported by us and the virtio device.
 186  */
 187 static int
 188 vhost_set_features(int vid, uint64_t *pu)
 189 {
 190         struct virtio_net *dev;
 191
 192         dev = get_device(vid);
 193         if (dev == NULL)
 194                 return -1;
 195         if (*pu & ~VHOST_FEATURES)
 196                 return -1;
 197
 198         dev->features = *pu;
 199         if (dev->features &
 200                 ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
 201                 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 202         } else {
 203                 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
 204         }
 205         LOG_DEBUG(VHOST_CONFIG,
 206                 "(%d) mergeable RX buffers %s, virtio 1 %s\n",
 207                 dev->vid,
 208                 (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
 209                 (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
 210
 211         return 0;
 212 }
 213
 214 /*
 215  * The virtio device sends us the size of the descriptor ring.
 216  */
 217 static int
 218 vhost_set_vring_num(int vid, struct vhost_vring_state *state)
 219 {
 220         struct virtio_net *dev;
 221
 222         dev = get_device(vid);
 223         if (dev == NULL)
 224                 return -1;
 225
 226         /* State->index refers to the queue index. The txq is 1, rxq is 0. */
 227         dev->virtqueue[state->index]->size = state->num;
 228
 229         return 0;
 230 }
 231
 232 /*
 233  * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the
 234  * same numa node as the memory of vring descriptor.
 235  */
 236 #ifdef RTE_LIBRTE_VHOST_NUMA
 237 static struct virtio_net*
 238 numa_realloc(struct virtio_net *dev, int index)
 239 {
 240         int oldnode, newnode;
 241         struct virtio_net *old_dev;
 242         struct vhost_virtqueue *old_vq, *vq;
 243         int ret;
 244
 245         /*
 246          * vq is allocated on pairs, we should try to do realloc
 247          * on first queue of one queue pair only.
 248          */
 249         if (index % VIRTIO_QNUM != 0)
 250                 return dev;
 251
 252         old_dev = dev;
 253         vq = old_vq = dev->virtqueue[index];
 254
 255         ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc,
 256                             MPOL_F_NODE | MPOL_F_ADDR);
 257
 258         /* check if we need to reallocate vq */
 259         ret |= get_mempolicy(&oldnode, NULL, 0, old_vq,
 260                              MPOL_F_NODE | MPOL_F_ADDR);
 261         if (ret) {
 262                 RTE_LOG(ERR, VHOST_CONFIG,
 263                         "Unable to get vq numa information.\n");
 264                 return dev;
 265         }
 266         if (oldnode != newnode) {
 267                 RTE_LOG(INFO, VHOST_CONFIG,
 268                         "reallocate vq from %d to %d node\n", oldnode, newnode);
 269                 vq = rte_malloc_socket(NULL, sizeof(*vq) * VIRTIO_QNUM, 0,
 270                                        newnode);
 271                 if (!vq)
 272                         return dev;
 273
 274                 memcpy(vq, old_vq, sizeof(*vq) * VIRTIO_QNUM);
 275                 rte_free(old_vq);
 276         }
 277
 278         /* check if we need to reallocate dev */
 279         ret = get_mempolicy(&oldnode, NULL, 0, old_dev,
 280                             MPOL_F_NODE | MPOL_F_ADDR);
 281         if (ret) {
 282                 RTE_LOG(ERR, VHOST_CONFIG,
 283                         "Unable to get dev numa information.\n");
 284                 goto out;
 285         }
 286         if (oldnode != newnode) {
 287                 RTE_LOG(INFO, VHOST_CONFIG,
 288                         "reallocate dev from %d to %d node\n",
 289                         oldnode, newnode);
 290                 dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode);
 291                 if (!dev) {
 292                         dev = old_dev;
 293                         goto out;
 294                 }
 295
 296                 memcpy(dev, old_dev, sizeof(*dev));
 297                 rte_free(old_dev);
 298         }
 299
 300 out:
 301         dev->virtqueue[index] = vq;
 302         dev->virtqueue[index + 1] = vq + 1;
 303         vhost_devices[dev->vid] = dev;
 304
 305         return dev;
 306 }
 307 #else
 308 static struct virtio_net*
 309 numa_realloc(struct virtio_net *dev, int index __rte_unused)
 310 {
 311         return dev;
 312 }
 313 #endif
 314
 315 /*
 316  * Converts QEMU virtual address to Vhost virtual address. This function is
 317  * used to convert the ring addresses to our address space.
 318  */
 319 static uint64_t
 320 qva_to_vva(struct virtio_net *dev, uint64_t qemu_va)
 321 {
 322         struct virtio_memory_regions *region;
 323         uint64_t vhost_va = 0;
 324         uint32_t regionidx = 0;
 325
 326         /* Find the region where the address lives. */
 327         for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
 328                 region = &dev->mem->regions[regionidx];
 329                 if ((qemu_va >= region->userspace_address) &&
 330                         (qemu_va <= region->userspace_address +
 331                         region->memory_size)) {
 332                         vhost_va = qemu_va + region->guest_phys_address +
 333                                 region->address_offset -
 334                                 region->userspace_address;
 335                         break;
 336                 }
 337         }
 338         return vhost_va;
 339 }
 340
 341 /*
 342  * The virtio device sends us the desc, used and avail ring addresses.
 343  * This function then converts these to our address space.
 344  */
 345 static int
 346 vhost_set_vring_addr(int vid, struct vhost_vring_addr *addr)
 347 {
 348         struct virtio_net *dev;
 349         struct vhost_virtqueue *vq;
 350
 351         dev = get_device(vid);
 352         if ((dev == NULL) || (dev->mem == NULL))
 353                 return -1;
 354
 355         /* addr->index refers to the queue index. The txq 1, rxq is 0. */
 356         vq = dev->virtqueue[addr->index];
 357
 358         /* The addresses are converted from QEMU virtual to Vhost virtual. */
 359         vq->desc = (struct vring_desc *)(uintptr_t)qva_to_vva(dev,
 360                         addr->desc_user_addr);
 361         if (vq->desc == 0) {
 362                 RTE_LOG(ERR, VHOST_CONFIG,
 363                         "(%d) failed to find desc ring address.\n",
 364                         dev->vid);
 365                 return -1;
 366         }
 367
 368         dev = numa_realloc(dev, addr->index);
 369         vq = dev->virtqueue[addr->index];
 370
 371         vq->avail = (struct vring_avail *)(uintptr_t)qva_to_vva(dev,
 372                         addr->avail_user_addr);
 373         if (vq->avail == 0) {
 374                 RTE_LOG(ERR, VHOST_CONFIG,
 375                         "(%d) failed to find avail ring address.\n",
 376                         dev->vid);
 377                 return -1;
 378         }
 379
 380         vq->used = (struct vring_used *)(uintptr_t)qva_to_vva(dev,
 381                         addr->used_user_addr);
 382         if (vq->used == 0) {
 383                 RTE_LOG(ERR, VHOST_CONFIG,
 384                         "(%d) failed to find used ring address.\n",
 385                         dev->vid);
 386                 return -1;
 387         }
 388
 389         if (vq->last_used_idx != vq->used->idx) {
 390                 RTE_LOG(WARNING, VHOST_CONFIG,
 391                         "last_used_idx (%u) and vq->used->idx (%u) mismatches; "
 392                         "some packets maybe resent for Tx and dropped for Rx\n",
 393                         vq->last_used_idx, vq->used->idx);
 394                 vq->last_used_idx     = vq->used->idx;
 395         }
 396
 397         vq->log_guest_addr = addr->log_guest_addr;
 398
 399         LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
 400                         dev->vid, vq->desc);
 401         LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n",
 402                         dev->vid, vq->avail);
 403         LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n",
 404                         dev->vid, vq->used);
 405         LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n",
 406                         dev->vid, vq->log_guest_addr);
 407
 408         return 0;
 409 }
 410
 411 /*
 412  * The virtio device sends us the available ring last used index.
 413  */
 414 static int
 415 vhost_set_vring_base(int vid, struct vhost_vring_state *state)
 416 {
 417         struct virtio_net *dev;
 418
 419         dev = get_device(vid);
 420         if (dev == NULL)
 421                 return -1;
 422
 423         /* State->index refers to the queue index. The txq is 1, rxq is 0. */
 424         dev->virtqueue[state->index]->last_used_idx = state->num;
 425
 426         return 0;
 427 }
 428
 429 static int
 430 user_set_mem_table(int vid, struct VhostUserMsg *pmsg)
 431 {
 432         struct VhostUserMemory memory = pmsg->payload.memory;
 433         struct virtio_memory_regions *pregion;
 434         uint64_t mapped_address, mapped_size;
 435         struct virtio_net *dev;
 436         unsigned int idx = 0;
 437         struct orig_region_map *pregion_orig;
 438         uint64_t alignment;
 439
 440         /* unmap old memory regions one by one*/
 441         dev = get_device(vid);
 442         if (dev == NULL)
 443                 return -1;
 444
 445         /* Remove from the data plane. */
 446         if (dev->flags & VIRTIO_DEV_RUNNING) {
 447                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 448                 notify_ops->destroy_device(vid);
 449         }
 450
 451         if (dev->mem) {
 452                 free_mem_region(dev);
 453                 free(dev->mem);
 454                 dev->mem = NULL;
 455         }
 456
 457         dev->mem = calloc(1,
 458                 sizeof(struct virtio_memory) +
 459                 sizeof(struct virtio_memory_regions) * memory.nregions +
 460                 sizeof(struct orig_region_map) * memory.nregions);
 461         if (dev->mem == NULL) {
 462                 RTE_LOG(ERR, VHOST_CONFIG,
 463                         "(%d) failed to allocate memory for dev->mem\n",
 464                         dev->vid);
 465                 return -1;
 466         }
 467         dev->mem->nregions = memory.nregions;
 468
 469         pregion_orig = orig_region(dev->mem, memory.nregions);
 470         for (idx = 0; idx < memory.nregions; idx++) {
 471                 pregion = &dev->mem->regions[idx];
 472                 pregion->guest_phys_address =
 473                         memory.regions[idx].guest_phys_addr;
 474                 pregion->guest_phys_address_end =
 475                         memory.regions[idx].guest_phys_addr +
 476                         memory.regions[idx].memory_size;
 477                 pregion->memory_size =
 478                         memory.regions[idx].memory_size;
 479                 pregion->userspace_address =
 480                         memory.regions[idx].userspace_addr;
 481
 482                 /* This is ugly */
 483                 mapped_size = memory.regions[idx].memory_size +
 484                         memory.regions[idx].mmap_offset;
 485
 486                 /* mmap() without flag of MAP_ANONYMOUS, should be called
 487                  * with length argument aligned with hugepagesz at older
 488                  * longterm version Linux, like 2.6.32 and 3.2.72, or
 489                  * mmap() will fail with EINVAL.
 490                  *
 491                  * to avoid failure, make sure in caller to keep length
 492                  * aligned.
 493                  */
 494                 alignment = get_blk_size(pmsg->fds[idx]);
 495                 if (alignment == (uint64_t)-1) {
 496                         RTE_LOG(ERR, VHOST_CONFIG,
 497                                 "couldn't get hugepage size through fstat\n");
 498                         goto err_mmap;
 499                 }
 500                 mapped_size = RTE_ALIGN_CEIL(mapped_size, alignment);
 501
 502                 mapped_address = (uint64_t)(uintptr_t)mmap(NULL,
 503                         mapped_size,
 504                         PROT_READ | PROT_WRITE, MAP_SHARED,
 505                         pmsg->fds[idx],
 506                         0);
 507
 508                 RTE_LOG(INFO, VHOST_CONFIG,
 509                         "mapped region %d fd:%d to:%p sz:0x%"PRIx64" "
 510                         "off:0x%"PRIx64" align:0x%"PRIx64"\n",
 511                         idx, pmsg->fds[idx], (void *)(uintptr_t)mapped_address,
 512                         mapped_size, memory.regions[idx].mmap_offset,
 513                         alignment);
 514
 515                 if (mapped_address == (uint64_t)(uintptr_t)MAP_FAILED) {
 516                         RTE_LOG(ERR, VHOST_CONFIG,
 517                                 "mmap qemu guest failed.\n");
 518                         goto err_mmap;
 519                 }
 520
 521                 pregion_orig[idx].mapped_address = mapped_address;
 522                 pregion_orig[idx].mapped_size = mapped_size;
 523                 pregion_orig[idx].blksz = alignment;
 524                 pregion_orig[idx].fd = pmsg->fds[idx];
 525
 526                 mapped_address +=  memory.regions[idx].mmap_offset;
 527
 528                 pregion->address_offset = mapped_address -
 529                         pregion->guest_phys_address;
 530
 531                 if (memory.regions[idx].guest_phys_addr == 0) {
 532                         dev->mem->base_address =
 533                                 memory.regions[idx].userspace_addr;
 534                         dev->mem->mapped_address =
 535                                 pregion->address_offset;
 536                 }
 537
 538                 LOG_DEBUG(VHOST_CONFIG,
 539                         "REGION: %u GPA: %p QEMU VA: %p SIZE (%"PRIu64")\n",
 540                         idx,
 541                         (void *)(uintptr_t)pregion->guest_phys_address,
 542                         (void *)(uintptr_t)pregion->userspace_address,
 543                          pregion->memory_size);
 544         }
 545
 546         return 0;
 547
 548 err_mmap:
 549         while (idx--) {
 550                 munmap((void *)(uintptr_t)pregion_orig[idx].mapped_address,
 551                                 pregion_orig[idx].mapped_size);
 552                 close(pregion_orig[idx].fd);
 553         }
 554         free(dev->mem);
 555         dev->mem = NULL;
 556         return -1;
 557 }
 558
 559 static int
 560 vq_is_ready(struct vhost_virtqueue *vq)
 561 {
 562         return vq && vq->desc   &&
 563                vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
 564                vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD;
 565 }
 566
 567 static int
 568 virtio_is_ready(struct virtio_net *dev)
 569 {
 570         struct vhost_virtqueue *rvq, *tvq;
 571         uint32_t i;
 572
 573         for (i = 0; i < dev->virt_qp_nb; i++) {
 574                 rvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ];
 575                 tvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ];
 576
 577                 if (!vq_is_ready(rvq) || !vq_is_ready(tvq)) {
 578                         RTE_LOG(INFO, VHOST_CONFIG,
 579                                 "virtio is not ready for processing.\n");
 580                         return 0;
 581                 }
 582         }
 583
 584         RTE_LOG(INFO, VHOST_CONFIG,
 585                 "virtio is now ready for processing.\n");
 586         return 1;
 587 }
 588
 589 static void
 590 user_set_vring_call(int vid, struct VhostUserMsg *pmsg)
 591 {
 592         struct vhost_vring_file file;
 593         struct virtio_net *dev = get_device(vid);
 594         struct vhost_virtqueue *vq;
 595         uint32_t cur_qp_idx;
 596
 597         if (!dev)
 598                 return;
 599
 600         file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
 601         if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
 602                 file.fd = VIRTIO_INVALID_EVENTFD;
 603         else
 604                 file.fd = pmsg->fds[0];
 605         RTE_LOG(INFO, VHOST_CONFIG,
 606                 "vring call idx:%d file:%d\n", file.index, file.fd);
 607
 608         /*
 609          * FIXME: VHOST_SET_VRING_CALL is the first per-vring message
 610          * we get, so we do vring queue pair allocation here.
 611          */
 612         cur_qp_idx = file.index / VIRTIO_QNUM;
 613         if (cur_qp_idx + 1 > dev->virt_qp_nb) {
 614                 if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0)
 615                         return;
 616         }
 617
 618         vq = dev->virtqueue[file.index];
 619         assert(vq != NULL);
 620
 621         if (vq->callfd >= 0)
 622                 close(vq->callfd);
 623
 624         vq->callfd = file.fd;
 625 }
 626
 627 /*
 628  *  In vhost-user, when we receive kick message, will test whether virtio
 629  *  device is ready for packet processing.
 630  */
 631 static void
 632 user_set_vring_kick(int vid, struct VhostUserMsg *pmsg)
 633 {
 634         struct vhost_vring_file file;
 635         struct virtio_net *dev = get_device(vid);
 636         struct vhost_virtqueue *vq;
 637
 638         if (!dev)
 639                 return;
 640
 641         file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
 642         if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
 643                 file.fd = VIRTIO_INVALID_EVENTFD;
 644         else
 645                 file.fd = pmsg->fds[0];
 646         RTE_LOG(INFO, VHOST_CONFIG,
 647                 "vring kick idx:%d file:%d\n", file.index, file.fd);
 648
 649         vq = dev->virtqueue[file.index];
 650         if (vq->kickfd >= 0)
 651                 close(vq->kickfd);
 652         vq->kickfd = file.fd;
 653
 654         if (virtio_is_ready(dev) && !(dev->flags & VIRTIO_DEV_RUNNING)) {
 655                 if (notify_ops->new_device(vid) == 0)
 656                         dev->flags |= VIRTIO_DEV_RUNNING;
 657         }
 658 }
 659
 660 /*
 661  * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
 662  */
 663 static int
 664 user_get_vring_base(int vid, struct vhost_vring_state *state)
 665 {
 666         struct virtio_net *dev = get_device(vid);
 667
 668         if (dev == NULL)
 669                 return -1;
 670         /* We have to stop the queue (virtio) if it is running. */
 671         if (dev->flags & VIRTIO_DEV_RUNNING) {
 672                 dev->flags &= ~VIRTIO_DEV_RUNNING;
 673                 notify_ops->destroy_device(vid);
 674         }
 675
 676         /* Here we are safe to get the last used index */
 677         state->num = dev->virtqueue[state->index]->last_used_idx;
 678
 679         RTE_LOG(INFO, VHOST_CONFIG,
 680                 "vring base idx:%d file:%d\n", state->index, state->num);
 681         /*
 682          * Based on current qemu vhost-user implementation, this message is
 683          * sent and only sent in vhost_vring_stop.
 684          * TODO: cleanup the vring, it isn't usable since here.
 685          */
 686         if (dev->virtqueue[state->index]->kickfd >= 0)
 687                 close(dev->virtqueue[state->index]->kickfd);
 688
 689         dev->virtqueue[state->index]->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
 690
 691         return 0;
 692 }
 693
 694 /*
 695  * when virtio queues are ready to work, qemu will send us to
 696  * enable the virtio queue pair.
 697  */
 698 static int
 699 user_set_vring_enable(int vid, struct vhost_vring_state *state)
 700 {
 701         struct virtio_net *dev;
 702         int enable = (int)state->num;
 703
 704         dev = get_device(vid);
 705         if (dev == NULL)
 706                 return -1;
 707
 708         RTE_LOG(INFO, VHOST_CONFIG,
 709                 "set queue enable: %d to qp idx: %d\n",
 710                 enable, state->index);
 711
 712         if (notify_ops->vring_state_changed)
 713                 notify_ops->vring_state_changed(vid, state->index, enable);
 714
 715         dev->virtqueue[state->index]->enabled = enable;
 716
 717         return 0;
 718 }
 719
 720 static void
 721 user_set_protocol_features(int vid, uint64_t protocol_features)
 722 {
 723         struct virtio_net *dev;
 724
 725         dev = get_device(vid);
 726         if (dev == NULL || protocol_features & ~VHOST_USER_PROTOCOL_FEATURES)
 727                 return;
 728
 729         dev->protocol_features = protocol_features;
 730 }
 731
 732 static int
 733 user_set_log_base(int vid, struct VhostUserMsg *msg)
 734 {
 735         struct virtio_net *dev;
 736         int fd = msg->fds[0];
 737         uint64_t size, off;
 738         void *addr;
 739
 740         dev = get_device(vid);
 741         if (!dev)
 742                 return -1;
 743
 744         if (fd < 0) {
 745                 RTE_LOG(ERR, VHOST_CONFIG, "invalid log fd: %d\n", fd);
 746                 return -1;
 747         }
 748
 749         if (msg->size != sizeof(VhostUserLog)) {
 750                 RTE_LOG(ERR, VHOST_CONFIG,
 751                         "invalid log base msg size: %"PRId32" != %d\n",
 752                         msg->size, (int)sizeof(VhostUserLog));
 753                 return -1;
 754         }
 755
 756         size = msg->payload.log.mmap_size;
 757         off  = msg->payload.log.mmap_offset;
 758         RTE_LOG(INFO, VHOST_CONFIG,
 759                 "log mmap size: %"PRId64", offset: %"PRId64"\n",
 760                 size, off);
 761
 762         /*
 763          * mmap from 0 to workaround a hugepage mmap bug: mmap will
 764          * fail when offset is not page size aligned.
 765          */
 766         addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
 767         close(fd);
 768         if (addr == MAP_FAILED) {
 769                 RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n");
 770                 return -1;
 771         }
 772
 773         /*
 774          * Free previously mapped log memory on occasionally
 775          * multiple VHOST_USER_SET_LOG_BASE.
 776          */
 777         if (dev->log_addr) {
 778                 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
 779         }
 780         dev->log_addr = (uint64_t)(uintptr_t)addr;
 781         dev->log_base = dev->log_addr + off;
 782         dev->log_size = size;
 783
 784         return 0;
 785 }
 786
 787 /*
 788  * An rarp packet is constructed and broadcasted to notify switches about
 789  * the new location of the migrated VM, so that packets from outside will
 790  * not be lost after migration.
 791  *
 792  * However, we don't actually "send" a rarp packet here, instead, we set
 793  * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
 794  */
 795 static int
 796 user_send_rarp(int vid, struct VhostUserMsg *msg)
 797 {
 798         struct virtio_net *dev;
 799         uint8_t *mac = (uint8_t *)&msg->payload.u64;
 800
 801         dev = get_device(vid);
 802         if (!dev)
 803                 return -1;
 804
 805         RTE_LOG(DEBUG, VHOST_CONFIG,
 806                 ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
 807                 mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
 808         memcpy(dev->mac.addr_bytes, mac, 6);
 809
 810         /*
 811          * Set the flag to inject a RARP broadcast packet at
 812          * rte_vhost_dequeue_burst().
 813          *
 814          * rte_smp_wmb() is for making sure the mac is copied
 815          * before the flag is set.
 816          */
 817         rte_smp_wmb();
 818         rte_atomic16_set(&dev->broadcast_rarp, 1);
 819
 820         return 0;
 821 }
 822
 823 /* return bytes# of read on success or negative val on failure. */
 824 static int
 825 read_vhost_message(int sockfd, struct VhostUserMsg *msg)
 826 {
 827         int ret;
 828
 829         ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
 830                 msg->fds, VHOST_MEMORY_MAX_NREGIONS);
 831         if (ret <= 0)
 832                 return ret;
 833
 834         if (msg && msg->size) {
 835                 if (msg->size > sizeof(msg->payload)) {
 836                         RTE_LOG(ERR, VHOST_CONFIG,
 837                                 "invalid msg size: %d\n", msg->size);
 838                         return -1;
 839                 }
 840                 ret = read(sockfd, &msg->payload, msg->size);
 841                 if (ret <= 0)
 842                         return ret;
 843                 if (ret != (int)msg->size) {
 844                         RTE_LOG(ERR, VHOST_CONFIG,
 845                                 "read control message failed\n");
 846                         return -1;
 847                 }
 848         }
 849
 850         return ret;
 851 }
 852
 853 static int
 854 send_vhost_message(int sockfd, struct VhostUserMsg *msg)
 855 {
 856         int ret;
 857
 858         if (!msg)
 859                 return 0;
 860
 861         msg->flags &= ~VHOST_USER_VERSION_MASK;
 862         msg->flags |= VHOST_USER_VERSION;
 863         msg->flags |= VHOST_USER_REPLY_MASK;
 864
 865         ret = send_fd_message(sockfd, (char *)msg,
 866                 VHOST_USER_HDR_SIZE + msg->size, NULL, 0);
 867
 868         return ret;
 869 }
 870
 871 int
 872 vhost_user_msg_handler(int vid, int fd)
 873 {
 874         struct VhostUserMsg msg;
 875         uint64_t features = 0;
 876         int ret;
 877
 878         ret = read_vhost_message(fd, &msg);
 879         if (ret <= 0 || msg.request >= VHOST_USER_MAX) {
 880                 if (ret < 0)
 881                         RTE_LOG(ERR, VHOST_CONFIG,
 882                                 "vhost read message failed\n");
 883                 else if (ret == 0)
 884                         RTE_LOG(INFO, VHOST_CONFIG,
 885                                 "vhost peer closed\n");
 886                 else
 887                         RTE_LOG(ERR, VHOST_CONFIG,
 888                                 "vhost read incorrect message\n");
 889
 890                 return -1;
 891         }
 892
 893         RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n",
 894                 vhost_message_str[msg.request]);
 895         switch (msg.request) {
 896         case VHOST_USER_GET_FEATURES:
 897                 ret = vhost_get_features(vid, &features);
 898                 msg.payload.u64 = features;
 899                 msg.size = sizeof(msg.payload.u64);
 900                 send_vhost_message(fd, &msg);
 901                 break;
 902         case VHOST_USER_SET_FEATURES:
 903                 features = msg.payload.u64;
 904                 vhost_set_features(vid, &features);
 905                 break;
 906
 907         case VHOST_USER_GET_PROTOCOL_FEATURES:
 908                 msg.payload.u64 = VHOST_USER_PROTOCOL_FEATURES;
 909                 msg.size = sizeof(msg.payload.u64);
 910                 send_vhost_message(fd, &msg);
 911                 break;
 912         case VHOST_USER_SET_PROTOCOL_FEATURES:
 913                 user_set_protocol_features(vid, msg.payload.u64);
 914                 break;
 915
 916         case VHOST_USER_SET_OWNER:
 917                 vhost_set_owner(vid);
 918                 break;
 919         case VHOST_USER_RESET_OWNER:
 920                 vhost_reset_owner(vid);
 921                 break;
 922
 923         case VHOST_USER_SET_MEM_TABLE:
 924                 user_set_mem_table(vid, &msg);
 925                 break;
 926
 927         case VHOST_USER_SET_LOG_BASE:
 928                 user_set_log_base(vid, &msg);
 929
 930                 /* it needs a reply */
 931                 msg.size = sizeof(msg.payload.u64);
 932                 send_vhost_message(fd, &msg);
 933                 break;
 934         case VHOST_USER_SET_LOG_FD:
 935                 close(msg.fds[0]);
 936                 RTE_LOG(INFO, VHOST_CONFIG, "not implemented.\n");
 937                 break;
 938
 939         case VHOST_USER_SET_VRING_NUM:
 940                 vhost_set_vring_num(vid, &msg.payload.state);
 941                 break;
 942         case VHOST_USER_SET_VRING_ADDR:
 943                 vhost_set_vring_addr(vid, &msg.payload.addr);
 944                 break;
 945         case VHOST_USER_SET_VRING_BASE:
 946                 vhost_set_vring_base(vid, &msg.payload.state);
 947                 break;
 948
 949         case VHOST_USER_GET_VRING_BASE:
 950                 ret = user_get_vring_base(vid, &msg.payload.state);
 951                 msg.size = sizeof(msg.payload.state);
 952                 send_vhost_message(fd, &msg);
 953                 break;
 954
 955         case VHOST_USER_SET_VRING_KICK:
 956                 user_set_vring_kick(vid, &msg);
 957                 break;
 958         case VHOST_USER_SET_VRING_CALL:
 959                 user_set_vring_call(vid, &msg);
 960                 break;
 961
 962         case VHOST_USER_SET_VRING_ERR:
 963                 if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK))
 964                         close(msg.fds[0]);
 965                 RTE_LOG(INFO, VHOST_CONFIG, "not implemented\n");
 966                 break;
 967
 968         case VHOST_USER_GET_QUEUE_NUM:
 969                 msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS;
 970                 msg.size = sizeof(msg.payload.u64);
 971                 send_vhost_message(fd, &msg);
 972                 break;
 973
 974         case VHOST_USER_SET_VRING_ENABLE:
 975                 user_set_vring_enable(vid, &msg.payload.state);
 976                 break;
 977         case VHOST_USER_SEND_RARP:
 978                 user_send_rarp(vid, &msg);
 979                 break;
 980
 981         default:
 982                 break;
 983
 984         }
 985
 986         return 0;
 987 }