drivers/net/virtio/virtio_user/vhost_user.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <sys/socket.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <unistd.h>
   9 #include <fcntl.h>
  10 #include <sys/un.h>
  11 #include <string.h>
  12 #include <errno.h>
  13
  14 #include <rte_alarm.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_fbarray.h>
  17
  18 #include "vhost.h"
  19 #include "virtio_user_dev.h"
  20
  21 struct vhost_user_data {
  22         int vhostfd;
  23         int listenfd;
  24         uint64_t protocol_features;
  25 };
  26
  27 #ifndef VHOST_USER_F_PROTOCOL_FEATURES
  28 #define VHOST_USER_F_PROTOCOL_FEATURES 30
  29 #endif
  30
  31 /** Protocol features. */
  32 #ifndef VHOST_USER_PROTOCOL_F_MQ
  33 #define VHOST_USER_PROTOCOL_F_MQ 0
  34 #endif
  35
  36 #ifndef VHOST_USER_PROTOCOL_F_REPLY_ACK
  37 #define VHOST_USER_PROTOCOL_F_REPLY_ACK 3
  38 #endif
  39
  40 #ifndef VHOST_USER_PROTOCOL_F_STATUS
  41 #define VHOST_USER_PROTOCOL_F_STATUS 16
  42 #endif
  43
  44 #define VHOST_USER_SUPPORTED_PROTOCOL_FEATURES          \
  45         (1ULL << VHOST_USER_PROTOCOL_F_MQ |             \
  46          1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK |      \
  47          1ULL << VHOST_USER_PROTOCOL_F_STATUS)
  48
  49 /* The version of the protocol we support */
  50 #define VHOST_USER_VERSION    0x1
  51
  52 #define VHOST_MEMORY_MAX_NREGIONS 8
  53 struct vhost_memory {
  54         uint32_t nregions;
  55         uint32_t padding;
  56         struct vhost_memory_region regions[VHOST_MEMORY_MAX_NREGIONS];
  57 };
  58
  59 enum vhost_user_request {
  60         VHOST_USER_NONE = 0,
  61         VHOST_USER_GET_FEATURES = 1,
  62         VHOST_USER_SET_FEATURES = 2,
  63         VHOST_USER_SET_OWNER = 3,
  64         VHOST_USER_RESET_OWNER = 4,
  65         VHOST_USER_SET_MEM_TABLE = 5,
  66         VHOST_USER_SET_LOG_BASE = 6,
  67         VHOST_USER_SET_LOG_FD = 7,
  68         VHOST_USER_SET_VRING_NUM = 8,
  69         VHOST_USER_SET_VRING_ADDR = 9,
  70         VHOST_USER_SET_VRING_BASE = 10,
  71         VHOST_USER_GET_VRING_BASE = 11,
  72         VHOST_USER_SET_VRING_KICK = 12,
  73         VHOST_USER_SET_VRING_CALL = 13,
  74         VHOST_USER_SET_VRING_ERR = 14,
  75         VHOST_USER_GET_PROTOCOL_FEATURES = 15,
  76         VHOST_USER_SET_PROTOCOL_FEATURES = 16,
  77         VHOST_USER_GET_QUEUE_NUM = 17,
  78         VHOST_USER_SET_VRING_ENABLE = 18,
  79         VHOST_USER_SET_STATUS = 39,
  80         VHOST_USER_GET_STATUS = 40,
  81         VHOST_USER_MAX
  82 };
  83
  84 struct vhost_user_msg {
  85         enum vhost_user_request request;
  86
  87 #define VHOST_USER_VERSION_MASK     0x3
  88 #define VHOST_USER_REPLY_MASK       (0x1 << 2)
  89 #define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
  90         uint32_t flags;
  91         uint32_t size; /* the following payload size */
  92         union {
  93 #define VHOST_USER_VRING_IDX_MASK   0xff
  94 #define VHOST_USER_VRING_NOFD_MASK  (0x1 << 8)
  95                 uint64_t u64;
  96                 struct vhost_vring_state state;
  97                 struct vhost_vring_addr addr;
  98                 struct vhost_memory memory;
  99         } payload;
 100 } __rte_packed;
 101
 102 #define VHOST_USER_HDR_SIZE offsetof(struct vhost_user_msg, payload.u64)
 103 #define VHOST_USER_PAYLOAD_SIZE \
 104         (sizeof(struct vhost_user_msg) - VHOST_USER_HDR_SIZE)
 105
 106 static int
 107 vhost_user_write(int fd, struct vhost_user_msg *msg, int *fds, int fd_num)
 108 {
 109         int r;
 110         struct msghdr msgh;
 111         struct iovec iov;
 112         size_t fd_size = fd_num * sizeof(int);
 113         char control[CMSG_SPACE(fd_size)];
 114         struct cmsghdr *cmsg;
 115
 116         memset(&msgh, 0, sizeof(msgh));
 117         memset(control, 0, sizeof(control));
 118
 119         iov.iov_base = (uint8_t *)msg;
 120         iov.iov_len = VHOST_USER_HDR_SIZE + msg->size;
 121
 122         msgh.msg_iov = &iov;
 123         msgh.msg_iovlen = 1;
 124         msgh.msg_control = control;
 125         msgh.msg_controllen = sizeof(control);
 126
 127         cmsg = CMSG_FIRSTHDR(&msgh);
 128         cmsg->cmsg_len = CMSG_LEN(fd_size);
 129         cmsg->cmsg_level = SOL_SOCKET;
 130         cmsg->cmsg_type = SCM_RIGHTS;
 131         memcpy(CMSG_DATA(cmsg), fds, fd_size);
 132
 133         do {
 134                 r = sendmsg(fd, &msgh, 0);
 135         } while (r < 0 && errno == EINTR);
 136
 137         if (r < 0)
 138                 PMD_DRV_LOG(ERR, "Failed to send msg: %s", strerror(errno));
 139
 140         return r;
 141 }
 142
 143 static int
 144 vhost_user_read(int fd, struct vhost_user_msg *msg)
 145 {
 146         uint32_t valid_flags = VHOST_USER_REPLY_MASK | VHOST_USER_VERSION;
 147         int ret, sz_hdr = VHOST_USER_HDR_SIZE, sz_payload;
 148
 149         ret = recv(fd, (void *)msg, sz_hdr, 0);
 150         if (ret < 0) {
 151                 PMD_DRV_LOG(ERR, "Failed to recv msg header: %s", strerror(errno));
 152                 return -1;
 153         } else if (ret < sz_hdr) {
 154                 PMD_DRV_LOG(ERR, "Failed to recv msg hdr: %d instead of %d.",
 155                             ret, sz_hdr);
 156                 return -1;
 157         }
 158
 159         /* validate msg flags */
 160         if (msg->flags != (valid_flags)) {
 161                 PMD_DRV_LOG(ERR, "Failed to recv msg: flags 0x%x instead of 0x%x.",
 162                             msg->flags, valid_flags);
 163                 return -1;
 164         }
 165
 166         sz_payload = msg->size;
 167
 168         if ((size_t)sz_payload > sizeof(msg->payload)) {
 169                 PMD_DRV_LOG(ERR, "Payload size overflow, header says %d but max %zu",
 170                                 sz_payload, sizeof(msg->payload));
 171                 return -1;
 172         }
 173
 174         if (sz_payload) {
 175                 ret = recv(fd, (void *)((char *)msg + sz_hdr), sz_payload, 0);
 176                 if (ret < 0) {
 177                         PMD_DRV_LOG(ERR, "Failed to recv msg payload: %s", strerror(errno));
 178                         return -1;
 179                 } else if (ret < sz_payload) {
 180                         PMD_DRV_LOG(ERR, "Failed to recv msg payload: %d instead of %u.",
 181                                 ret, msg->size);
 182                         return -1;
 183                 }
 184         }
 185
 186         return 0;
 187 }
 188
 189 static int
 190 vhost_user_check_reply_ack(struct virtio_user_dev *dev, struct vhost_user_msg *msg)
 191 {
 192         struct vhost_user_data *data = dev->backend_data;
 193         enum vhost_user_request req = msg->request;
 194         int ret;
 195
 196         if (!(msg->flags & VHOST_USER_NEED_REPLY_MASK))
 197                 return 0;
 198
 199         ret = vhost_user_read(data->vhostfd, msg);
 200         if (ret < 0) {
 201                 PMD_DRV_LOG(ERR, "Failed to read reply-ack");
 202                 return -1;
 203         }
 204
 205         if (req != msg->request) {
 206                 PMD_DRV_LOG(ERR, "Unexpected reply-ack request type (%d)", msg->request);
 207                 return -1;
 208         }
 209
 210         if (msg->size != sizeof(msg->payload.u64)) {
 211                 PMD_DRV_LOG(ERR, "Unexpected reply-ack payload size (%u)", msg->size);
 212                 return -1;
 213         }
 214
 215         if (msg->payload.u64) {
 216                 PMD_DRV_LOG(ERR, "Slave replied NACK to request type (%d)", msg->request);
 217                 return -1;
 218         }
 219
 220         return 0;
 221 }
 222
 223 static int
 224 vhost_user_set_owner(struct virtio_user_dev *dev)
 225 {
 226         int ret;
 227         struct vhost_user_data *data = dev->backend_data;
 228         struct vhost_user_msg msg = {
 229                 .request = VHOST_USER_SET_OWNER,
 230                 .flags = VHOST_USER_VERSION,
 231         };
 232
 233         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 234         if (ret < 0) {
 235                 PMD_DRV_LOG(ERR, "Failed to set owner");
 236                 return -1;
 237         }
 238
 239         return 0;
 240 }
 241
 242 static int
 243 vhost_user_get_protocol_features(struct virtio_user_dev *dev, uint64_t *features)
 244 {
 245         int ret;
 246         struct vhost_user_data *data = dev->backend_data;
 247         struct vhost_user_msg msg = {
 248                 .request = VHOST_USER_GET_PROTOCOL_FEATURES,
 249                 .flags = VHOST_USER_VERSION,
 250         };
 251
 252         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 253         if (ret < 0)
 254                 goto err;
 255
 256         ret = vhost_user_read(data->vhostfd, &msg);
 257         if (ret < 0)
 258                 goto err;
 259
 260         if (msg.request != VHOST_USER_GET_PROTOCOL_FEATURES) {
 261                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 262                 goto err;
 263         }
 264
 265         if (msg.size != sizeof(*features)) {
 266                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 267                 goto err;
 268         }
 269
 270         *features = msg.payload.u64;
 271
 272         return 0;
 273 err:
 274         PMD_DRV_LOG(ERR, "Failed to get backend protocol features");
 275
 276         return -1;
 277 }
 278
 279 static int
 280 vhost_user_set_protocol_features(struct virtio_user_dev *dev, uint64_t features)
 281 {
 282         int ret;
 283         struct vhost_user_data *data = dev->backend_data;
 284         struct vhost_user_msg msg = {
 285                 .request = VHOST_USER_SET_PROTOCOL_FEATURES,
 286                 .flags = VHOST_USER_VERSION,
 287                 .size = sizeof(features),
 288                 .payload.u64 = features,
 289         };
 290
 291         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 292         if (ret < 0) {
 293                 PMD_DRV_LOG(ERR, "Failed to set protocol features");
 294                 return -1;
 295         }
 296
 297         return 0;
 298 }
 299
 300 static int
 301 vhost_user_get_features(struct virtio_user_dev *dev, uint64_t *features)
 302 {
 303         int ret;
 304         struct vhost_user_data *data = dev->backend_data;
 305         struct vhost_user_msg msg = {
 306                 .request = VHOST_USER_GET_FEATURES,
 307                 .flags = VHOST_USER_VERSION,
 308         };
 309
 310         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 311         if (ret < 0)
 312                 goto err;
 313
 314         ret = vhost_user_read(data->vhostfd, &msg);
 315         if (ret < 0)
 316                 goto err;
 317
 318         if (msg.request != VHOST_USER_GET_FEATURES) {
 319                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 320                 goto err;
 321         }
 322
 323         if (msg.size != sizeof(*features)) {
 324                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 325                 goto err;
 326         }
 327
 328         *features = msg.payload.u64;
 329
 330         if (!(*features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)))
 331                 return 0;
 332
 333         /* Negotiate protocol features */
 334         ret = vhost_user_get_protocol_features(dev, &data->protocol_features);
 335         if (ret < 0)
 336                 goto err;
 337
 338         data->protocol_features &= VHOST_USER_SUPPORTED_PROTOCOL_FEATURES;
 339
 340         ret = vhost_user_set_protocol_features(dev, data->protocol_features);
 341         if (ret < 0)
 342                 goto err;
 343
 344         if (!(data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)))
 345                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);
 346
 347         return 0;
 348 err:
 349         PMD_DRV_LOG(ERR, "Failed to get backend features");
 350
 351         return -1;
 352 }
 353
 354 static int
 355 vhost_user_set_features(struct virtio_user_dev *dev, uint64_t features)
 356 {
 357         int ret;
 358         struct vhost_user_data *data = dev->backend_data;
 359         struct vhost_user_msg msg = {
 360                 .request = VHOST_USER_SET_FEATURES,
 361                 .flags = VHOST_USER_VERSION,
 362                 .size = sizeof(features),
 363                 .payload.u64 = features,
 364         };
 365
 366         msg.payload.u64 |= dev->device_features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES);
 367
 368         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 369         if (ret < 0) {
 370                 PMD_DRV_LOG(ERR, "Failed to set features");
 371                 return -1;
 372         }
 373
 374         return 0;
 375 }
 376
 377 struct walk_arg {
 378         struct vhost_memory *vm;
 379         int *fds;
 380         int region_nr;
 381 };
 382
 383 static int
 384 update_memory_region(const struct rte_memseg_list *msl __rte_unused,
 385                 const struct rte_memseg *ms, void *arg)
 386 {
 387         struct walk_arg *wa = arg;
 388         struct vhost_memory_region *mr;
 389         uint64_t start_addr, end_addr;
 390         size_t offset;
 391         int i, fd;
 392
 393         fd = rte_memseg_get_fd_thread_unsafe(ms);
 394         if (fd < 0) {
 395                 PMD_DRV_LOG(ERR, "Failed to get fd, ms=%p rte_errno=%d",
 396                         ms, rte_errno);
 397                 return -1;
 398         }
 399
 400         if (rte_memseg_get_fd_offset_thread_unsafe(ms, &offset) < 0) {
 401                 PMD_DRV_LOG(ERR, "Failed to get offset, ms=%p rte_errno=%d",
 402                         ms, rte_errno);
 403                 return -1;
 404         }
 405
 406         start_addr = (uint64_t)(uintptr_t)ms->addr;
 407         end_addr = start_addr + ms->len;
 408
 409         for (i = 0; i < wa->region_nr; i++) {
 410                 if (wa->fds[i] != fd)
 411                         continue;
 412
 413                 mr = &wa->vm->regions[i];
 414
 415                 if (mr->userspace_addr + mr->memory_size < end_addr)
 416                         mr->memory_size = end_addr - mr->userspace_addr;
 417
 418                 if (mr->userspace_addr > start_addr) {
 419                         mr->userspace_addr = start_addr;
 420                         mr->guest_phys_addr = start_addr;
 421                 }
 422
 423                 if (mr->mmap_offset > offset)
 424                         mr->mmap_offset = offset;
 425
 426                 PMD_DRV_LOG(DEBUG, "index=%d fd=%d offset=0x%" PRIx64
 427                         " addr=0x%" PRIx64 " len=%" PRIu64, i, fd,
 428                         mr->mmap_offset, mr->userspace_addr,
 429                         mr->memory_size);
 430
 431                 return 0;
 432         }
 433
 434         if (i >= VHOST_MEMORY_MAX_NREGIONS) {
 435                 PMD_DRV_LOG(ERR, "Too many memory regions");
 436                 return -1;
 437         }
 438
 439         mr = &wa->vm->regions[i];
 440         wa->fds[i] = fd;
 441
 442         mr->guest_phys_addr = start_addr;
 443         mr->userspace_addr = start_addr;
 444         mr->memory_size = ms->len;
 445         mr->mmap_offset = offset;
 446
 447         PMD_DRV_LOG(DEBUG, "index=%d fd=%d offset=0x%" PRIx64
 448                 " addr=0x%" PRIx64 " len=%" PRIu64, i, fd,
 449                 mr->mmap_offset, mr->userspace_addr,
 450                 mr->memory_size);
 451
 452         wa->region_nr++;
 453
 454         return 0;
 455 }
 456
 457 static int
 458 vhost_user_set_memory_table(struct virtio_user_dev *dev)
 459 {
 460         struct walk_arg wa;
 461         int fds[VHOST_MEMORY_MAX_NREGIONS];
 462         int ret, fd_num;
 463         struct vhost_user_data *data = dev->backend_data;
 464         struct vhost_user_msg msg = {
 465                 .request = VHOST_USER_SET_MEM_TABLE,
 466                 .flags = VHOST_USER_VERSION,
 467         };
 468
 469         if (data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK))
 470                 msg.flags |= VHOST_USER_NEED_REPLY_MASK;
 471
 472         wa.region_nr = 0;
 473         wa.vm = &msg.payload.memory;
 474         wa.fds = fds;
 475
 476         /*
 477          * The memory lock has already been taken by memory subsystem
 478          * or virtio_user_start_device().
 479          */
 480         ret = rte_memseg_walk_thread_unsafe(update_memory_region, &wa);
 481         if (ret < 0)
 482                 goto err;
 483
 484         fd_num = wa.region_nr;
 485         msg.payload.memory.nregions = wa.region_nr;
 486         msg.payload.memory.padding = 0;
 487
 488         msg.size = sizeof(msg.payload.memory.nregions);
 489         msg.size += sizeof(msg.payload.memory.padding);
 490         msg.size += fd_num * sizeof(struct vhost_memory_region);
 491
 492         ret = vhost_user_write(data->vhostfd, &msg, fds, fd_num);
 493         if (ret < 0)
 494                 goto err;
 495
 496         return vhost_user_check_reply_ack(dev, &msg);
 497 err:
 498         PMD_DRV_LOG(ERR, "Failed to set memory table");
 499         return -1;
 500 }
 501
 502 static int
 503 vhost_user_set_vring(struct virtio_user_dev *dev, enum vhost_user_request req,
 504                 struct vhost_vring_state *state)
 505 {
 506         int ret;
 507         struct vhost_user_data *data = dev->backend_data;
 508         struct vhost_user_msg msg = {
 509                 .request = req,
 510                 .flags = VHOST_USER_VERSION,
 511                 .size = sizeof(*state),
 512                 .payload.state = *state,
 513         };
 514
 515         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 516         if (ret < 0) {
 517                 PMD_DRV_LOG(ERR, "Failed to set vring state (request %d)", req);
 518                 return -1;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int
 525 vhost_user_set_vring_enable(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 526 {
 527         return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, state);
 528 }
 529
 530 static int
 531 vhost_user_set_vring_num(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 532 {
 533         return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_NUM, state);
 534 }
 535
 536 static int
 537 vhost_user_set_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 538 {
 539         return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_BASE, state);
 540 }
 541
 542 static int
 543 vhost_user_get_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 544 {
 545         int ret;
 546         struct vhost_user_msg msg;
 547         struct vhost_user_data *data = dev->backend_data;
 548         unsigned int index = state->index;
 549
 550         ret = vhost_user_set_vring(dev, VHOST_USER_GET_VRING_BASE, state);
 551         if (ret < 0) {
 552                 PMD_DRV_LOG(ERR, "Failed to send request");
 553                 goto err;
 554         }
 555
 556         ret = vhost_user_read(data->vhostfd, &msg);
 557         if (ret < 0) {
 558                 PMD_DRV_LOG(ERR, "Failed to read reply");
 559                 goto err;
 560         }
 561
 562         if (msg.request != VHOST_USER_GET_VRING_BASE) {
 563                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 564                 goto err;
 565         }
 566
 567         if (msg.size != sizeof(*state)) {
 568                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 569                 goto err;
 570         }
 571
 572         if (msg.payload.state.index != index) {
 573                 PMD_DRV_LOG(ERR, "Unexpected ring index (%u)", state->index);
 574                 goto err;
 575         }
 576
 577         *state = msg.payload.state;
 578
 579         return 0;
 580 err:
 581         PMD_DRV_LOG(ERR, "Failed to get vring base");
 582         return -1;
 583 }
 584
 585 static int
 586 vhost_user_set_vring_file(struct virtio_user_dev *dev, enum vhost_user_request req,
 587                 struct vhost_vring_file *file)
 588 {
 589         int ret;
 590         int fd = file->fd;
 591         int num_fd = 0;
 592         struct vhost_user_data *data = dev->backend_data;
 593         struct vhost_user_msg msg = {
 594                 .request = req,
 595                 .flags = VHOST_USER_VERSION,
 596                 .size = sizeof(msg.payload.u64),
 597                 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
 598         };
 599
 600         if (fd >= 0)
 601                 num_fd++;
 602         else
 603                 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
 604
 605         ret = vhost_user_write(data->vhostfd, &msg, &fd, num_fd);
 606         if (ret < 0) {
 607                 PMD_DRV_LOG(ERR, "Failed to set vring file (request %d)", req);
 608                 return -1;
 609         }
 610
 611         return 0;
 612 }
 613
 614 static int
 615 vhost_user_set_vring_call(struct virtio_user_dev *dev, struct vhost_vring_file *file)
 616 {
 617         return vhost_user_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
 618 }
 619
 620 static int
 621 vhost_user_set_vring_kick(struct virtio_user_dev *dev, struct vhost_vring_file *file)
 622 {
 623         return vhost_user_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
 624 }
 625
 626
 627 static int
 628 vhost_user_set_vring_addr(struct virtio_user_dev *dev, struct vhost_vring_addr *addr)
 629 {
 630         int ret;
 631         struct vhost_user_data *data = dev->backend_data;
 632         struct vhost_user_msg msg = {
 633                 .request = VHOST_USER_SET_VRING_ADDR,
 634                 .flags = VHOST_USER_VERSION,
 635                 .size = sizeof(*addr),
 636                 .payload.addr = *addr,
 637         };
 638
 639         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 640         if (ret < 0) {
 641                 PMD_DRV_LOG(ERR, "Failed to send vring addresses");
 642                 return -1;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static int
 649 vhost_user_get_status(struct virtio_user_dev *dev, uint8_t *status)
 650 {
 651         int ret;
 652         struct vhost_user_data *data = dev->backend_data;
 653         struct vhost_user_msg msg = {
 654                 .request = VHOST_USER_GET_STATUS,
 655                 .flags = VHOST_USER_VERSION,
 656         };
 657
 658         /*
 659          * If features have not been negotiated, we don't know if the backend
 660          * supports protocol features
 661          */
 662         if (!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
 663                 return -ENOTSUP;
 664
 665         /* Status protocol feature requires protocol features support */
 666         if (!(dev->device_features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)))
 667                 return -ENOTSUP;
 668
 669         if (!(data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS)))
 670                 return -ENOTSUP;
 671
 672         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 673         if (ret < 0) {
 674                 PMD_DRV_LOG(ERR, "Failed to send request");
 675                 goto err;
 676         }
 677
 678         ret = vhost_user_read(data->vhostfd, &msg);
 679         if (ret < 0) {
 680                 PMD_DRV_LOG(ERR, "Failed to recv request");
 681                 goto err;
 682         }
 683
 684         if (msg.request != VHOST_USER_GET_STATUS) {
 685                 PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
 686                 goto err;
 687         }
 688
 689         if (msg.size != sizeof(msg.payload.u64)) {
 690                 PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
 691                 goto err;
 692         }
 693
 694         *status = (uint8_t)msg.payload.u64;
 695
 696         return 0;
 697 err:
 698         PMD_DRV_LOG(ERR, "Failed to get device status");
 699         return -1;
 700 }
 701
 702 static int
 703 vhost_user_set_status(struct virtio_user_dev *dev, uint8_t status)
 704 {
 705         int ret;
 706         struct vhost_user_data *data = dev->backend_data;
 707         struct vhost_user_msg msg = {
 708                 .request = VHOST_USER_SET_STATUS,
 709                 .flags = VHOST_USER_VERSION,
 710                 .size = sizeof(msg.payload.u64),
 711                 .payload.u64 = status,
 712         };
 713
 714         /*
 715          * If features have not been negotiated, we don't know if the backend
 716          * supports protocol features
 717          */
 718         if (!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
 719                 return -ENOTSUP;
 720
 721         /* Status protocol feature requires protocol features support */
 722         if (!(dev->device_features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)))
 723                 return -ENOTSUP;
 724
 725         if (!(data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_STATUS)))
 726                 return -ENOTSUP;
 727
 728         if (data->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK))
 729                 msg.flags |= VHOST_USER_NEED_REPLY_MASK;
 730
 731         ret = vhost_user_write(data->vhostfd, &msg, NULL, 0);
 732         if (ret < 0) {
 733                 PMD_DRV_LOG(ERR, "Failed to send get status request");
 734                 return -1;
 735         }
 736
 737         return vhost_user_check_reply_ack(dev, &msg);
 738 }
 739
 740 #define MAX_VIRTIO_USER_BACKLOG 1
 741 static int
 742 vhost_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un)
 743 {
 744         int ret;
 745         int flag;
 746         struct vhost_user_data *data = dev->backend_data;
 747         int fd = data->listenfd;
 748
 749         ret = bind(fd, (struct sockaddr *)un, sizeof(*un));
 750         if (ret < 0) {
 751                 PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again",
 752                             dev->path, strerror(errno));
 753                 return -1;
 754         }
 755         ret = listen(fd, MAX_VIRTIO_USER_BACKLOG);
 756         if (ret < 0)
 757                 return -1;
 758
 759         PMD_DRV_LOG(NOTICE, "(%s) waiting for client connection...", dev->path);
 760         data->vhostfd = accept(fd, NULL, NULL);
 761         if (data->vhostfd < 0) {
 762                 PMD_DRV_LOG(ERR, "Failed to accept initial client connection (%s)",
 763                                 strerror(errno));
 764                 return -1;
 765         }
 766
 767         flag = fcntl(fd, F_GETFL);
 768         if (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0) {
 769                 PMD_DRV_LOG(ERR, "fcntl failed, %s", strerror(errno));
 770                 return -1;
 771         }
 772
 773         return 0;
 774 }
 775
 776 static int
 777 vhost_user_server_disconnect(struct virtio_user_dev *dev)
 778 {
 779         struct vhost_user_data *data = dev->backend_data;
 780
 781         if (data->vhostfd < 0) {
 782                 PMD_DRV_LOG(ERR, "(%s) Expected valid Vhost FD", dev->path);
 783                 return -1;
 784         }
 785
 786         close(data->vhostfd);
 787         data->vhostfd = -1;
 788
 789         return 0;
 790 }
 791
 792 static int
 793 vhost_user_server_reconnect(struct virtio_user_dev *dev)
 794 {
 795         struct vhost_user_data *data = dev->backend_data;
 796         int fd;
 797
 798         fd = accept(data->listenfd, NULL, NULL);
 799         if (fd < 0)
 800                 return -1;
 801
 802         data->vhostfd = fd;
 803
 804         return 0;
 805 }
 806
 807 /**
 808  * Set up environment to talk with a vhost user backend.
 809  *
 810  * @return
 811  *   - (-1) if fail;
 812  *   - (0) if succeed.
 813  */
 814 static int
 815 vhost_user_setup(struct virtio_user_dev *dev)
 816 {
 817         int fd;
 818         int flag;
 819         struct sockaddr_un un;
 820         struct vhost_user_data *data;
 821
 822         data = malloc(sizeof(*data));
 823         if (!data) {
 824                 PMD_DRV_LOG(ERR, "(%s) Failed to allocate Vhost-user data", dev->path);
 825                 return -1;
 826         }
 827
 828         memset(data, 0, sizeof(*data));
 829
 830         dev->backend_data = data;
 831
 832         data->vhostfd = -1;
 833         data->listenfd = -1;
 834
 835         fd = socket(AF_UNIX, SOCK_STREAM, 0);
 836         if (fd < 0) {
 837                 PMD_DRV_LOG(ERR, "socket() error, %s", strerror(errno));
 838                 goto err_data;
 839         }
 840
 841         flag = fcntl(fd, F_GETFD);
 842         if (flag == -1)
 843                 PMD_DRV_LOG(WARNING, "fcntl get fd failed, %s", strerror(errno));
 844         else if (fcntl(fd, F_SETFD, flag | FD_CLOEXEC) < 0)
 845                 PMD_DRV_LOG(WARNING, "fcntl set fd failed, %s", strerror(errno));
 846
 847         memset(&un, 0, sizeof(un));
 848         un.sun_family = AF_UNIX;
 849         strlcpy(un.sun_path, dev->path, sizeof(un.sun_path));
 850
 851         if (dev->is_server) {
 852                 data->listenfd = fd;
 853                 if (vhost_user_start_server(dev, &un) < 0) {
 854                         PMD_DRV_LOG(ERR, "virtio-user startup fails in server mode");
 855                         goto err_socket;
 856                 }
 857         } else {
 858                 if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
 859                         PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
 860                         goto err_socket;
 861                 }
 862                 data->vhostfd = fd;
 863         }
 864
 865         return 0;
 866
 867 err_socket:
 868         close(fd);
 869 err_data:
 870         free(data);
 871         dev->backend_data = NULL;
 872
 873         return -1;
 874 }
 875
 876 static int
 877 vhost_user_destroy(struct virtio_user_dev *dev)
 878 {
 879         struct vhost_user_data *data = dev->backend_data;
 880
 881         if (!data)
 882                 return 0;
 883
 884         if (data->vhostfd >= 0) {
 885                 close(data->vhostfd);
 886                 data->vhostfd = -1;
 887         }
 888
 889         if (data->listenfd >= 0) {
 890                 close(data->listenfd);
 891                 data->listenfd = -1;
 892         }
 893
 894         free(data);
 895         dev->backend_data = NULL;
 896
 897         return 0;
 898 }
 899
 900 static int
 901 vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
 902                              uint16_t pair_idx,
 903                              int enable)
 904 {
 905         struct vhost_user_data *data = dev->backend_data;
 906         int i;
 907
 908         if (data->vhostfd < 0)
 909                 return 0;
 910
 911         if (dev->qp_enabled[pair_idx] == enable)
 912                 return 0;
 913
 914         for (i = 0; i < 2; ++i) {
 915                 struct vhost_vring_state state = {
 916                         .index = pair_idx * 2 + i,
 917                         .num = enable,
 918                 };
 919
 920                 if (vhost_user_set_vring_enable(dev, &state))
 921                         return -1;
 922         }
 923
 924         dev->qp_enabled[pair_idx] = enable;
 925         return 0;
 926 }
 927
 928 static int
 929 vhost_user_get_backend_features(uint64_t *features)
 930 {
 931         *features = 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
 932
 933         return 0;
 934 }
 935
 936 static int
 937 vhost_user_update_link_state(struct virtio_user_dev *dev)
 938 {
 939         struct vhost_user_data *data = dev->backend_data;
 940         char buf[128];
 941
 942         if (data->vhostfd >= 0) {
 943                 int r;
 944                 int flags;
 945
 946                 flags = fcntl(data->vhostfd, F_GETFL);
 947                 if (fcntl(data->vhostfd, F_SETFL, flags | O_NONBLOCK) == -1) {
 948                         PMD_DRV_LOG(ERR, "error setting O_NONBLOCK flag");
 949                         return -1;
 950                 }
 951
 952                 r = recv(data->vhostfd, buf, 128, MSG_PEEK);
 953                 if (r == 0 || (r < 0 && errno != EAGAIN)) {
 954                         dev->net_status &= (~VIRTIO_NET_S_LINK_UP);
 955                         PMD_DRV_LOG(ERR, "virtio-user port %u is down", dev->hw.port_id);
 956
 957                         /* This function could be called in the process
 958                          * of interrupt handling, callback cannot be
 959                          * unregistered here, set an alarm to do it.
 960                          */
 961                         rte_eal_alarm_set(1,
 962                                 virtio_user_dev_delayed_disconnect_handler,
 963                                 (void *)dev);
 964                 } else {
 965                         dev->net_status |= VIRTIO_NET_S_LINK_UP;
 966                 }
 967
 968                 if (fcntl(data->vhostfd, F_SETFL,
 969                                         flags & ~O_NONBLOCK) == -1) {
 970                         PMD_DRV_LOG(ERR, "error clearing O_NONBLOCK flag");
 971                         return -1;
 972                 }
 973         } else if (dev->is_server) {
 974                 dev->net_status &= (~VIRTIO_NET_S_LINK_UP);
 975                 if (virtio_user_dev_server_reconnect(dev) >= 0)
 976                         dev->net_status |= VIRTIO_NET_S_LINK_UP;
 977         }
 978
 979         return 0;
 980 }
 981
 982 static int
 983 vhost_user_get_intr_fd(struct virtio_user_dev *dev)
 984 {
 985         struct vhost_user_data *data = dev->backend_data;
 986
 987         if (dev->is_server && data->vhostfd == -1)
 988                 return data->listenfd;
 989
 990         return data->vhostfd;
 991 }
 992
 993 struct virtio_user_backend_ops virtio_ops_user = {
 994         .setup = vhost_user_setup,
 995         .destroy = vhost_user_destroy,
 996         .get_backend_features = vhost_user_get_backend_features,
 997         .set_owner = vhost_user_set_owner,
 998         .get_features = vhost_user_get_features,
 999         .set_features = vhost_user_set_features,
1000         .set_memory_table = vhost_user_set_memory_table,
1001         .set_vring_num = vhost_user_set_vring_num,
1002         .set_vring_base = vhost_user_set_vring_base,
1003         .get_vring_base = vhost_user_get_vring_base,
1004         .set_vring_call = vhost_user_set_vring_call,
1005         .set_vring_kick = vhost_user_set_vring_kick,
1006         .set_vring_addr = vhost_user_set_vring_addr,
1007         .get_status = vhost_user_get_status,
1008         .set_status = vhost_user_set_status,
1009         .enable_qp = vhost_user_enable_queue_pair,
1010         .update_link_state = vhost_user_update_link_state,
1011         .server_disconnect = vhost_user_server_disconnect,
1012         .server_reconnect = vhost_user_server_reconnect,
1013         .get_intr_fd = vhost_user_get_intr_fd,
1014 };