drivers/net/virtio/virtio_user/vhost_user.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <sys/socket.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <unistd.h>
   9 #include <fcntl.h>
  10 #include <sys/un.h>
  11 #include <string.h>
  12 #include <errno.h>
  13
  14 #include <rte_alarm.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_fbarray.h>
  17
  18 #include "vhost.h"
  19 #include "virtio_user_dev.h"
  20
  21 struct vhost_user_data {
  22         int vhostfd;
  23         int listenfd;
  24         uint64_t protocol_features;
  25 };
  26
  27 #ifndef VHOST_USER_F_PROTOCOL_FEATURES
  28 #define VHOST_USER_F_PROTOCOL_FEATURES 30
  29 #endif
  30
  31 /** Protocol features. */
  32 #ifndef VHOST_USER_PROTOCOL_F_MQ
  33 #define VHOST_USER_PROTOCOL_F_MQ 0
  34 #endif
  35
  36 #ifndef VHOST_USER_PROTOCOL_F_REPLY_ACK
  37 #define VHOST_USER_PROTOCOL_F_REPLY_ACK 3
  38 #endif
  39
  40 #ifndef VHOST_USER_PROTOCOL_F_STATUS
  41 #define VHOST_USER_PROTOCOL_F_STATUS 16
  42 #endif
  43
  44 #define VHOST_USER_SUPPORTED_PROTOCOL_FEATURES          \
  45         (1ULL << VHOST_USER_PROTOCOL_F_MQ |             \
  46          1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK |      \
  47          1ULL << VHOST_USER_PROTOCOL_F_STATUS)
  48
  49 /* The version of the protocol we support */
  50 #define VHOST_USER_VERSION    0x1
  51
  52 #define VHOST_MEMORY_MAX_NREGIONS 8
  53 struct vhost_memory {
  54         uint32_t nregions;
  55         uint32_t padding;
  56         struct vhost_memory_region regions[VHOST_MEMORY_MAX_NREGIONS];
  57 };
  58
  59 enum vhost_user_request {
  60         VHOST_USER_NONE = 0,
  61         VHOST_USER_GET_FEATURES = 1,
  62         VHOST_USER_SET_FEATURES = 2,
  63         VHOST_USER_SET_OWNER = 3,
  64         VHOST_USER_RESET_OWNER = 4,
  65         VHOST_USER_SET_MEM_TABLE = 5,
  66         VHOST_USER_SET_LOG_BASE = 6,
  67         VHOST_USER_SET_LOG_FD = 7,
  68         VHOST_USER_SET_VRING_NUM = 8,
  69         VHOST_USER_SET_VRING_ADDR = 9,
  70         VHOST_USER_SET_VRING_BASE = 10,
  71         VHOST_USER_GET_VRING_BASE = 11,
  72         VHOST_USER_SET_VRING_KICK = 12,
  73         VHOST_USER_SET_VRING_CALL = 13,
  74         VHOST_USER_SET_VRING_ERR = 14,
  75         VHOST_USER_GET_PROTOCOL_FEATURES = 15,
  76         VHOST_USER_SET_PROTOCOL_FEATURES = 16,
  77         VHOST_USER_GET_QUEUE_NUM = 17,
  78         VHOST_USER_SET_VRING_ENABLE = 18,
  79         VHOST_USER_SET_STATUS = 39,
  80         VHOST_USER_GET_STATUS = 40,
  81         VHOST_USER_MAX
  82 };
  83
  84 struct vhost_user_msg {
  85         enum vhost_user_request request;
  86
  87 #define VHOST_USER_VERSION_MASK     0x3
  88 #define VHOST_USER_REPLY_MASK       (0x1 << 2)
  89 #define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
  90         uint32_t flags;
  91         uint32_t size; /* the following payload size */
  92         union {
  93 #define VHOST_USER_VRING_IDX_MASK   0xff
  94 #define VHOST_USER_VRING_NOFD_MASK  (0x1 << 8)
  95                 uint64_t u64;
  96                 struct vhost_vring_state state;
  97                 struct vhost_vring_addr addr;
  98                 struct vhost_memory memory;
  99         } payload;
 100         int fds[VHOST_MEMORY_MAX_NREGIONS];
 101 } __rte_packed;
 102
 103 #define VHOST_USER_HDR_SIZE offsetof(struct vhost_user_msg, payload.u64)
 104 #define VHOST_USER_PAYLOAD_SIZE \
 105         (sizeof(struct vhost_user_msg) - VHOST_USER_HDR_SIZE)
 106
 107 static int
 108 vhost_user_write(int fd, struct vhost_user_msg *msg, int *fds, int fd_num)
 109 {
 110         int r;
 111         struct msghdr msgh;
 112         struct iovec iov;
 113         size_t fd_size = fd_num * sizeof(int);
 114         char control[CMSG_SPACE(fd_size)];
 115         struct cmsghdr *cmsg;
 116
 117         memset(&msgh, 0, sizeof(msgh));
 118         memset(control, 0, sizeof(control));
 119
 120         iov.iov_base = (uint8_t *)msg;
 121         iov.iov_len = VHOST_USER_HDR_SIZE + msg->size;
 122
 123         msgh.msg_iov = &iov;
 124         msgh.msg_iovlen = 1;
 125         msgh.msg_control = control;
 126         msgh.msg_controllen = sizeof(control);
 127
 128         cmsg = CMSG_FIRSTHDR(&msgh);
 129         cmsg->cmsg_len = CMSG_LEN(fd_size);
 130         cmsg->cmsg_level = SOL_SOCKET;
 131         cmsg->cmsg_type = SCM_RIGHTS;
 132         memcpy(CMSG_DATA(cmsg), fds, fd_size);
 133
 134         do {
 135                 r = sendmsg(fd, &msgh, 0);
 136         } while (r < 0 && errno == EINTR);
 137
 138         if (r < 0)
 139                 PMD_DRV_LOG(ERR, "Failed to send msg: %s", strerror(errno));
 140
 141         return r;
 142 }
 143
 144 static int
 145 vhost_user_read(int fd, struct vhost_user_msg *msg)
 146 {
 147         uint32_t valid_flags = VHOST_USER_REPLY_MASK | VHOST_USER_VERSION;
 148         int ret, sz_hdr = VHOST_USER_HDR_SIZE, sz_payload;
 149
 150         ret = recv(fd, (void *)msg, sz_hdr, 0);
 151         if (ret < sz_hdr) {
 152                 PMD_DRV_LOG(ERR, "Failed to recv msg hdr: %d instead of %d.",
 153                             ret, sz_hdr);
 154                 goto fail;
 155         }
 156
 157         /* validate msg flags */
 158         if (msg->flags != (valid_flags)) {
 159                 PMD_DRV_LOG(ERR, "Failed to recv msg: flags %x instead of %x.",
 160                             msg->flags, valid_flags);
 161                 goto fail;
 162         }
 163
 164         sz_payload = msg->size;
 165
 166         if ((size_t)sz_payload > sizeof(msg->payload))
 167                 goto fail;
 168
 169         if (sz_payload) {
 170                 ret = recv(fd, (void *)((char *)msg + sz_hdr), sz_payload, 0);
 171                 if (ret < sz_payload) {
 172                         PMD_DRV_LOG(ERR,
 173                                 "Failed to recv msg payload: %d instead of %d.",
 174                                 ret, msg->size);
 175                         goto fail;
 176                 }
 177         }
 178
 179         return 0;
 180
 181 fail:
 182         return -1;
 183 }
 184
 185 static int
 186 vhost_user_check_reply_ack(struct virtio_user_dev *dev, struct vhost_user_msg *msg)
 187 {
 188         struct vhost_user_data *data = dev->backend_data;
 189         enum vhost_user_request req = msg->request;
 190         int ret;
 191
 192         if (!(msg->flags & VHOST_USER_NEED_REPLY_MASK))
 193                 return 0;
 194
 195         ret = vhost_user_read(data->vhostfd, msg);
 196         if (ret < 0) {
 197                 PMD_DRV_LOG(ERR, "Failed to read reply-ack");
 198                 return -1;
 199         }
 200
 201         if (req != msg->request) {
 202                 PMD_DRV_LOG(ERR, "Unexpected reply-ack request type (%d)", msg->request);
 203                 return -1;
 204         }
 205
 206         if (msg->size != sizeof(msg->payload.u64)) {
 207                 PMD_DRV_LOG(ERR, "Unexpected reply-ack payload size (%u)", msg->size);
 208                 return -1;
 209         }
 210
 211         if (msg->payload.u64) {
 212                 PMD_DRV_LOG(ERR, "Slave replied NACK to request type (%d)", msg->request);
 213                 return -1;
 214         }
 215
 216         return 0;
 217 }
 218
 219 static int
 220 vhost_user_set_owner(struct virtio_user_dev *dev)
 221 {
 222         int ret;
 223         struct vhost_user_data *data = dev->backend_data;
 224         struct vhost_user_msg msg = {
 225                 .request = VHOST_USER_SET_OWNER,
 226                 .flags = VHOST_USER_VERSION,
 227         };
 228
 229         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 230         if (ret < 0) {
 231                 PMD_DRV_LOG(ERR, "Failed to set owner");
 232                 return -1;
 233         }
 234
 235         return 0;
 236 }
 237
 238 static int
 239 vhost_user_get_protocol_features(struct virtio_user_dev *dev, uint64_t *features)
 240 {
 241         int ret;
 242         struct vhost_user_data *data = dev->backend_data;
 243         struct vhost_user_msg msg = {
 244                 .request = VHOST_USER_GET_PROTOCOL_FEATURES,
 245                 .flags = VHOST_USER_VERSION,
 246         };
 247
 248         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 249         if (ret < 0)
 250                 goto err;
 251
 252         ret = vhost_user_read(data->vhostfd, &msg);
 253         if (ret < 0)
 254                 goto err;
 255
 256         if (msg.request != VHOST_USER_GET_PROTOCOL_FEATURES) {
 257                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 258                 goto err;
 259         }
 260
 261         if (msg.size != sizeof(*features)) {
 262                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 263                 goto err;
 264         }
 265
 266         *features = msg.payload.u64;
 267
 268         return 0;
 269 err:
 270         PMD_DRV_LOG(ERR, "Failed to get backend protocol features");
 271
 272         return -1;
 273 }
 274
 275 static int
 276 vhost_user_set_protocol_features(struct virtio_user_dev *dev, uint64_t features)
 277 {
 278         int ret;
 279         struct vhost_user_data *data = dev->backend_data;
 280         struct vhost_user_msg msg = {
 281                 .request = VHOST_USER_SET_PROTOCOL_FEATURES,
 282                 .flags = VHOST_USER_VERSION,
 283                 .size = sizeof(features),
 284                 .payload.u64 = features,
 285         };
 286
 287         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 288         if (ret < 0) {
 289                 PMD_DRV_LOG(ERR, "Failed to set protocol features");
 290                 return -1;
 291         }
 292
 293         return 0;
 294 }
 295
 296 static int
 297 vhost_user_get_features(struct virtio_user_dev *dev, uint64_t *features)
 298 {
 299         int ret;
 300         struct vhost_user_data *data = dev->backend_data;
 301         struct vhost_user_msg msg = {
 302                 .request = VHOST_USER_GET_FEATURES,
 303                 .flags = VHOST_USER_VERSION,
 304         };
 305
 306         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 307         if (ret < 0)
 308                 goto err;
 309
 310         ret = vhost_user_read(data->vhostfd, &msg);
 311         if (ret < 0)
 312                 goto err;
 313
 314         if (msg.request != VHOST_USER_GET_FEATURES) {
 315                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 316                 goto err;
 317         }
 318
 319         if (msg.size != sizeof(*features)) {
 320                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 321                 goto err;
 322         }
 323
 324         *features = msg.payload.u64;
 325
 326         if (!(*features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)))
 327                 return 0;
 328
 329         /* Negotiate protocol features */
 330         ret = vhost_user_get_protocol_features(dev, &data->protocol_features);
 331         if (ret < 0)
 332                 goto err;
 333
 334         data->protocol_features &= VHOST_USER_SUPPORTED_PROTOCOL_FEATURES;
 335
 336         ret = vhost_user_set_protocol_features(dev, data->protocol_features);
 337         if (ret < 0)
 338                 goto err;
 339
 340         if (!(data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)))
 341                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);
 342
 343         return 0;
 344 err:
 345         PMD_DRV_LOG(ERR, "Failed to get backend features");
 346
 347         return -1;
 348 }
 349
 350 static int
 351 vhost_user_set_features(struct virtio_user_dev *dev, uint64_t features)
 352 {
 353         int ret;
 354         struct vhost_user_data *data = dev->backend_data;
 355         struct vhost_user_msg msg = {
 356                 .request = VHOST_USER_SET_FEATURES,
 357                 .flags = VHOST_USER_VERSION,
 358                 .size = sizeof(features),
 359                 .payload.u64 = features,
 360         };
 361
 362         msg.payload.u64 |= dev->device_features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES);
 363
 364         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 365         if (ret < 0) {
 366                 PMD_DRV_LOG(ERR, "Failed to set features");
 367                 return -1;
 368         }
 369
 370         return 0;
 371 }
 372
 373 struct walk_arg {
 374         struct vhost_memory *vm;
 375         int *fds;
 376         int region_nr;
 377 };
 378
 379 static int
 380 update_memory_region(const struct rte_memseg_list *msl __rte_unused,
 381                 const struct rte_memseg *ms, void *arg)
 382 {
 383         struct walk_arg *wa = arg;
 384         struct vhost_memory_region *mr;
 385         uint64_t start_addr, end_addr;
 386         size_t offset;
 387         int i, fd;
 388
 389         fd = rte_memseg_get_fd_thread_unsafe(ms);
 390         if (fd < 0) {
 391                 PMD_DRV_LOG(ERR, "Failed to get fd, ms=%p rte_errno=%d",
 392                         ms, rte_errno);
 393                 return -1;
 394         }
 395
 396         if (rte_memseg_get_fd_offset_thread_unsafe(ms, &offset) < 0) {
 397                 PMD_DRV_LOG(ERR, "Failed to get offset, ms=%p rte_errno=%d",
 398                         ms, rte_errno);
 399                 return -1;
 400         }
 401
 402         start_addr = (uint64_t)(uintptr_t)ms->addr;
 403         end_addr = start_addr + ms->len;
 404
 405         for (i = 0; i < wa->region_nr; i++) {
 406                 if (wa->fds[i] != fd)
 407                         continue;
 408
 409                 mr = &wa->vm->regions[i];
 410
 411                 if (mr->userspace_addr + mr->memory_size < end_addr)
 412                         mr->memory_size = end_addr - mr->userspace_addr;
 413
 414                 if (mr->userspace_addr > start_addr) {
 415                         mr->userspace_addr = start_addr;
 416                         mr->guest_phys_addr = start_addr;
 417                 }
 418
 419                 if (mr->mmap_offset > offset)
 420                         mr->mmap_offset = offset;
 421
 422                 PMD_DRV_LOG(DEBUG, "index=%d fd=%d offset=0x%" PRIx64
 423                         " addr=0x%" PRIx64 " len=%" PRIu64, i, fd,
 424                         mr->mmap_offset, mr->userspace_addr,
 425                         mr->memory_size);
 426
 427                 return 0;
 428         }
 429
 430         if (i >= VHOST_MEMORY_MAX_NREGIONS) {
 431                 PMD_DRV_LOG(ERR, "Too many memory regions");
 432                 return -1;
 433         }
 434
 435         mr = &wa->vm->regions[i];
 436         wa->fds[i] = fd;
 437
 438         mr->guest_phys_addr = start_addr;
 439         mr->userspace_addr = start_addr;
 440         mr->memory_size = ms->len;
 441         mr->mmap_offset = offset;
 442
 443         PMD_DRV_LOG(DEBUG, "index=%d fd=%d offset=0x%" PRIx64
 444                 " addr=0x%" PRIx64 " len=%" PRIu64, i, fd,
 445                 mr->mmap_offset, mr->userspace_addr,
 446                 mr->memory_size);
 447
 448         wa->region_nr++;
 449
 450         return 0;
 451 }
 452
 453 static int
 454 vhost_user_set_memory_table(struct virtio_user_dev *dev)
 455 {
 456         struct walk_arg wa;
 457         int fds[VHOST_MEMORY_MAX_NREGIONS];
 458         int ret, fd_num;
 459         struct vhost_user_data *data = dev->backend_data;
 460         struct vhost_user_msg msg = {
 461                 .request = VHOST_USER_SET_MEM_TABLE,
 462                 .flags = VHOST_USER_VERSION,
 463         };
 464
 465         if (data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK))
 466                 msg.flags |= VHOST_USER_NEED_REPLY_MASK;
 467
 468         wa.region_nr = 0;
 469         wa.vm = &msg.payload.memory;
 470         wa.fds = fds;
 471
 472         /*
 473          * The memory lock has already been taken by memory subsystem
 474          * or virtio_user_start_device().
 475          */
 476         ret = rte_memseg_walk_thread_unsafe(update_memory_region, &wa);
 477         if (ret < 0)
 478                 goto err;
 479
 480         fd_num = wa.region_nr;
 481         msg.payload.memory.nregions = wa.region_nr;
 482         msg.payload.memory.padding = 0;
 483
 484         msg.size = sizeof(msg.payload.memory.nregions);
 485         msg.size += sizeof(msg.payload.memory.padding);
 486         msg.size += fd_num * sizeof(struct vhost_memory_region);
 487
 488         ret = vhost_user_write(data->vhostfd, &msg, fds, fd_num);
 489         if (ret < 0)
 490                 goto err;
 491
 492         return vhost_user_check_reply_ack(dev, &msg);
 493 err:
 494         PMD_DRV_LOG(ERR, "Failed to set memory table");
 495         return -1;
 496 }
 497
 498 static int
 499 vhost_user_set_vring(struct virtio_user_dev *dev, enum vhost_user_request req,
 500                 struct vhost_vring_state *state)
 501 {
 502         int ret;
 503         struct vhost_user_data *data = dev->backend_data;
 504         struct vhost_user_msg msg = {
 505                 .request = req,
 506                 .flags = VHOST_USER_VERSION,
 507                 .size = sizeof(*state),
 508                 .payload.state = *state,
 509         };
 510
 511         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 512         if (ret < 0) {
 513                 PMD_DRV_LOG(ERR, "Failed to set vring state (request %d)", req);
 514                 return -1;
 515         }
 516
 517         return 0;
 518 }
 519
 520 static int
 521 vhost_user_set_vring_enable(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 522 {
 523         return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, state);
 524 }
 525
 526 static int
 527 vhost_user_set_vring_num(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 528 {
 529         return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_NUM, state);
 530 }
 531
 532 static int
 533 vhost_user_set_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 534 {
 535         return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_BASE, state);
 536 }
 537
 538 static int
 539 vhost_user_get_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 540 {
 541         int ret;
 542         struct vhost_user_msg msg;
 543         struct vhost_user_data *data = dev->backend_data;
 544         unsigned int index = state->index;
 545
 546         ret = vhost_user_set_vring(dev, VHOST_USER_GET_VRING_BASE, state);
 547         if (ret < 0) {
 548                 PMD_DRV_LOG(ERR, "Failed to send request");
 549                 goto err;
 550         }
 551
 552         ret = vhost_user_read(data->vhostfd, &msg);
 553         if (ret < 0) {
 554                 PMD_DRV_LOG(ERR, "Failed to read reply");
 555                 goto err;
 556         }
 557
 558         if (msg.request != VHOST_USER_GET_VRING_BASE) {
 559                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 560                 goto err;
 561         }
 562
 563         if (msg.size != sizeof(*state)) {
 564                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 565                 goto err;
 566         }
 567
 568         if (msg.payload.state.index != index) {
 569                 PMD_DRV_LOG(ERR, "Unexpected ring index (%u)", state->index);
 570                 goto err;
 571         }
 572
 573         *state = msg.payload.state;
 574
 575         return 0;
 576 err:
 577         PMD_DRV_LOG(ERR, "Failed to get vring base");
 578         return -1;
 579 }
 580
 581 static int
 582 vhost_user_set_vring_file(struct virtio_user_dev *dev, enum vhost_user_request req,
 583                 struct vhost_vring_file *file)
 584 {
 585         int ret;
 586         int fd = file->fd;
 587         int num_fd = 0;
 588         struct vhost_user_data *data = dev->backend_data;
 589         struct vhost_user_msg msg = {
 590                 .request = req,
 591                 .flags = VHOST_USER_VERSION,
 592                 .size = sizeof(msg.payload.u64),
 593                 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
 594         };
 595
 596         if (fd >= 0)
 597                 num_fd++;
 598         else
 599                 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
 600
 601         ret = vhost_user_write(data->vhostfd, &msg, &fd, num_fd);
 602         if (ret < 0) {
 603                 PMD_DRV_LOG(ERR, "Failed to set vring file (request %d)", req);
 604                 return -1;
 605         }
 606
 607         return 0;
 608 }
 609
 610 static int
 611 vhost_user_set_vring_call(struct virtio_user_dev *dev, struct vhost_vring_file *file)
 612 {
 613         return vhost_user_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
 614 }
 615
 616 static int
 617 vhost_user_set_vring_kick(struct virtio_user_dev *dev, struct vhost_vring_file *file)
 618 {
 619         return vhost_user_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
 620 }
 621
 622
 623 static int
 624 vhost_user_set_vring_addr(struct virtio_user_dev *dev, struct vhost_vring_addr *addr)
 625 {
 626         int ret;
 627         struct vhost_user_data *data = dev->backend_data;
 628         struct vhost_user_msg msg = {
 629                 .request = VHOST_USER_SET_VRING_ADDR,
 630                 .flags = VHOST_USER_VERSION,
 631                 .size = sizeof(*addr),
 632                 .payload.addr = *addr,
 633         };
 634
 635         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 636         if (ret < 0) {
 637                 PMD_DRV_LOG(ERR, "Failed to send vring addresses");
 638                 return -1;
 639         }
 640
 641         return 0;
 642 }
 643
 644 static int
 645 vhost_user_get_status(struct virtio_user_dev *dev, uint8_t *status)
 646 {
 647         int ret;
 648         struct vhost_user_data *data = dev->backend_data;
 649         struct vhost_user_msg msg = {
 650                 .request = VHOST_USER_GET_STATUS,
 651                 .flags = VHOST_USER_VERSION,
 652         };
 653
 654         /*
 655          * If features have not been negotiated, we don't know if the backend
 656          * supports protocol features
 657          */
 658         if (!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
 659                 return -ENOTSUP;
 660
 661         /* Status protocol feature requires protocol features support */
 662         if (!(dev->device_features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)))
 663                 return -ENOTSUP;
 664
 665         if (!(data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS)))
 666                 return -ENOTSUP;
 667
 668         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 669         if (ret < 0) {
 670                 PMD_DRV_LOG(ERR, "Failed to send request");
 671                 goto err;
 672         }
 673
 674         ret = vhost_user_read(data->vhostfd, &msg);
 675         if (ret < 0) {
 676                 PMD_DRV_LOG(ERR, "Failed to recv request");
 677                 goto err;
 678         }
 679
 680         if (msg.request != VHOST_USER_GET_STATUS) {
 681                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 682                 goto err;
 683         }
 684
 685         if (msg.size != sizeof(msg.payload.u64)) {
 686                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 687                 goto err;
 688         }
 689
 690         *status = (uint8_t)msg.payload.u64;
 691
 692         return 0;
 693 err:
 694         PMD_DRV_LOG(ERR, "Failed to get device status");
 695         return -1;
 696 }
 697
 698 static int
 699 vhost_user_set_status(struct virtio_user_dev *dev, uint8_t status)
 700 {
 701         int ret;
 702         struct vhost_user_data *data = dev->backend_data;
 703         struct vhost_user_msg msg = {
 704                 .request = VHOST_USER_SET_STATUS,
 705                 .flags = VHOST_USER_VERSION,
 706                 .size = sizeof(msg.payload.u64),
 707                 .payload.u64 = status,
 708         };
 709
 710         /*
 711          * If features have not been negotiated, we don't know if the backend
 712          * supports protocol features
 713          */
 714         if (!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
 715                 return -ENOTSUP;
 716
 717         /* Status protocol feature requires protocol features support */
 718         if (!(dev->device_features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)))
 719                 return -ENOTSUP;
 720
 721         if (!(data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS)))
 722                 return -ENOTSUP;
 723
 724         if (data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK))
 725                 msg.flags |= VHOST_USER_NEED_REPLY_MASK;
 726
 727         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 728         if (ret < 0) {
 729                 PMD_DRV_LOG(ERR, "Failed to send get status request");
 730                 return -1;
 731         }
 732
 733         return vhost_user_check_reply_ack(dev, &msg);
 734 }
 735
 736 #define MAX_VIRTIO_USER_BACKLOG 1
 737 static int
 738 vhost_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un)
 739 {
 740         int ret;
 741         int flag;
 742         struct vhost_user_data *data = dev->backend_data;
 743         int fd = data->listenfd;
 744
 745         ret = bind(fd, (struct sockaddr *)un, sizeof(*un));
 746         if (ret < 0) {
 747                 PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again\n",
 748                             dev->path, strerror(errno));
 749                 return -1;
 750         }
 751         ret = listen(fd, MAX_VIRTIO_USER_BACKLOG);
 752         if (ret < 0)
 753                 return -1;
 754
 755         PMD_DRV_LOG(NOTICE, "(%s) waiting for client connection...", dev->path);
 756         data->vhostfd = accept(fd, NULL, NULL);
 757         if (data->vhostfd < 0) {
 758                 PMD_DRV_LOG(ERR, "Failed to accept initial client connection (%s)",
 759                                 strerror(errno));
 760                 return -1;
 761         }
 762
 763         flag = fcntl(fd, F_GETFL);
 764         if (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0) {
 765                 PMD_DRV_LOG(ERR, "fcntl failed, %s", strerror(errno));
 766                 return -1;
 767         }
 768
 769         return 0;
 770 }
 771
 772 static int
 773 vhost_user_server_disconnect(struct virtio_user_dev *dev)
 774 {
 775         struct vhost_user_data *data = dev->backend_data;
 776
 777         if (data->vhostfd < 0) {
 778                 PMD_DRV_LOG(ERR, "(%s) Expected valid Vhost FD", dev->path);
 779                 return -1;
 780         }
 781
 782         close(data->vhostfd);
 783         data->vhostfd = -1;
 784
 785         return 0;
 786 }
 787
 788 static int
 789 vhost_user_server_reconnect(struct virtio_user_dev *dev)
 790 {
 791         struct vhost_user_data *data = dev->backend_data;
 792         int fd;
 793
 794         fd = accept(data->listenfd, NULL, NULL);
 795         if (fd < 0)
 796                 return -1;
 797
 798         data->vhostfd = fd;
 799
 800         return 0;
 801 }
 802
 803 /**
 804  * Set up environment to talk with a vhost user backend.
 805  *
 806  * @return
 807  *   - (-1) if fail;
 808  *   - (0) if succeed.
 809  */
 810 static int
 811 vhost_user_setup(struct virtio_user_dev *dev)
 812 {
 813         int fd;
 814         int flag;
 815         struct sockaddr_un un;
 816         struct vhost_user_data *data;
 817
 818         data = malloc(sizeof(*data));
 819         if (!data) {
 820                 PMD_DRV_LOG(ERR, "(%s) Failed to allocate Vhost-user data\n", dev->path);
 821                 return -1;
 822         }
 823
 824         memset(data, 0, sizeof(*data));
 825
 826         dev->backend_data = data;
 827
 828         data->vhostfd = -1;
 829
 830         fd = socket(AF_UNIX, SOCK_STREAM, 0);
 831         if (fd < 0) {
 832                 PMD_DRV_LOG(ERR, "socket() error, %s", strerror(errno));
 833                 goto err_data;
 834         }
 835
 836         flag = fcntl(fd, F_GETFD);
 837         if (fcntl(fd, F_SETFD, flag | FD_CLOEXEC) < 0)
 838                 PMD_DRV_LOG(WARNING, "fcntl failed, %s", strerror(errno));
 839
 840         memset(&un, 0, sizeof(un));
 841         un.sun_family = AF_UNIX;
 842         strlcpy(un.sun_path, dev->path, sizeof(un.sun_path));
 843
 844         if (dev->is_server) {
 845                 data->listenfd = fd;
 846                 if (vhost_user_start_server(dev, &un) < 0) {
 847                         PMD_DRV_LOG(ERR, "virtio-user startup fails in server mode");
 848                         goto err_socket;
 849                 }
 850         } else {
 851                 if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
 852                         PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
 853                         goto err_socket;
 854                 }
 855                 data->vhostfd = fd;
 856         }
 857
 858         return 0;
 859
 860 err_socket:
 861         close(fd);
 862 err_data:
 863         free(data);
 864         dev->backend_data = NULL;
 865
 866         return -1;
 867 }
 868
 869 static int
 870 vhost_user_destroy(struct virtio_user_dev *dev)
 871 {
 872         struct vhost_user_data *data = dev->backend_data;
 873
 874         if (!data)
 875                 return 0;
 876
 877         if (data->vhostfd >= 0) {
 878                 close(data->vhostfd);
 879                 data->vhostfd = -1;
 880         }
 881
 882         if (data->listenfd >= 0) {
 883                 close(data->listenfd);
 884                 data->listenfd = -1;
 885         }
 886
 887         free(data);
 888         dev->backend_data = NULL;
 889
 890         return 0;
 891 }
 892
 893 static int
 894 vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
 895                              uint16_t pair_idx,
 896                              int enable)
 897 {
 898         struct vhost_user_data *data = dev->backend_data;
 899         int i;
 900
 901         if (data->vhostfd < 0)
 902                 return 0;
 903
 904         if (dev->qp_enabled[pair_idx] == enable)
 905                 return 0;
 906
 907         for (i = 0; i < 2; ++i) {
 908                 struct vhost_vring_state state = {
 909                         .index = pair_idx * 2 + i,
 910                         .num = enable,
 911                 };
 912
 913                 if (vhost_user_set_vring_enable(dev, &state))
 914                         return -1;
 915         }
 916
 917         dev->qp_enabled[pair_idx] = enable;
 918         return 0;
 919 }
 920
 921 static int
 922 vhost_user_get_backend_features(uint64_t *features)
 923 {
 924         *features = 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
 925
 926         return 0;
 927 }
 928
 929 static int
 930 vhost_user_update_link_state(struct virtio_user_dev *dev)
 931 {
 932         struct vhost_user_data *data = dev->backend_data;
 933         char buf[128];
 934
 935         if (data->vhostfd >= 0) {
 936                 int r;
 937                 int flags;
 938
 939                 flags = fcntl(data->vhostfd, F_GETFL);
 940                 if (fcntl(data->vhostfd, F_SETFL, flags | O_NONBLOCK) == -1) {
 941                         PMD_DRV_LOG(ERR, "error setting O_NONBLOCK flag");
 942                         return -1;
 943                 }
 944
 945                 r = recv(data->vhostfd, buf, 128, MSG_PEEK);
 946                 if (r == 0 || (r < 0 && errno != EAGAIN)) {
 947                         dev->net_status &= (~VIRTIO_NET_S_LINK_UP);
 948                         PMD_DRV_LOG(ERR, "virtio-user port %u is down", dev->port_id);
 949
 950                         /* This function could be called in the process
 951                          * of interrupt handling, callback cannot be
 952                          * unregistered here, set an alarm to do it.
 953                          */
 954                         rte_eal_alarm_set(1, virtio_user_dev_delayed_handler, (void *)dev);
 955                 } else {
 956                         dev->net_status |= VIRTIO_NET_S_LINK_UP;
 957                 }
 958
 959                 if (fcntl(data->vhostfd, F_SETFL,
 960                                         flags & ~O_NONBLOCK) == -1) {
 961                         PMD_DRV_LOG(ERR, "error clearing O_NONBLOCK flag");
 962                         return -1;
 963                 }
 964         } else if (dev->is_server) {
 965                 dev->net_status &= (~VIRTIO_NET_S_LINK_UP);
 966                 if (virtio_user_dev_server_reconnect(dev) >= 0)
 967                         dev->net_status |= VIRTIO_NET_S_LINK_UP;
 968         }
 969
 970         return 0;
 971 }
 972
 973 static int
 974 vhost_user_get_intr_fd(struct virtio_user_dev *dev)
 975 {
 976         struct vhost_user_data *data = dev->backend_data;
 977
 978         if (dev->is_server && data->vhostfd == -1)
 979                 return data->listenfd;
 980
 981         return data->vhostfd;
 982 }
 983
 984 struct virtio_user_backend_ops virtio_ops_user = {
 985         .setup = vhost_user_setup,
 986         .destroy = vhost_user_destroy,
 987         .get_backend_features = vhost_user_get_backend_features,
 988         .set_owner = vhost_user_set_owner,
 989         .get_features = vhost_user_get_features,
 990         .set_features = vhost_user_set_features,
 991         .set_memory_table = vhost_user_set_memory_table,
 992         .set_vring_num = vhost_user_set_vring_num,
 993         .set_vring_base = vhost_user_set_vring_base,
 994         .get_vring_base = vhost_user_get_vring_base,
 995         .set_vring_call = vhost_user_set_vring_call,
 996         .set_vring_kick = vhost_user_set_vring_kick,
 997         .set_vring_addr = vhost_user_set_vring_addr,
 998         .get_status = vhost_user_get_status,
 999         .set_status = vhost_user_set_status,
1000         .enable_qp = vhost_user_enable_queue_pair,
1001         .update_link_state = vhost_user_update_link_state,
1002         .server_disconnect = vhost_user_server_disconnect,
1003         .server_reconnect = vhost_user_server_reconnect,
1004         .get_intr_fd = vhost_user_get_intr_fd,
1005 };