lib/librte_vhost/socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdio.h>
   7 #include <limits.h>
   8 #include <stdlib.h>
   9 #include <unistd.h>
  10 #include <string.h>
  11 #include <sys/types.h>
  12 #include <sys/socket.h>
  13 #include <sys/un.h>
  14 #include <sys/queue.h>
  15 #include <errno.h>
  16 #include <fcntl.h>
  17 #include <pthread.h>
  18
  19 #include <rte_log.h>
  20
  21 #include "fd_man.h"
  22 #include "vhost.h"
  23 #include "vhost_user.h"
  24
  25
  26 TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
  27
  28 /*
  29  * Every time rte_vhost_driver_register() is invoked, an associated
  30  * vhost_user_socket struct will be created.
  31  */
  32 struct vhost_user_socket {
  33         struct vhost_user_connection_list conn_list;
  34         pthread_mutex_t conn_mutex;
  35         char *path;
  36         int socket_fd;
  37         struct sockaddr_un un;
  38         bool is_server;
  39         bool reconnect;
  40         bool dequeue_zero_copy;
  41         bool iommu_support;
  42         bool use_builtin_virtio_net;
  43         bool extbuf;
  44         bool linearbuf;
  45         bool async_copy;
  46
  47         /*
  48          * The "supported_features" indicates the feature bits the
  49          * vhost driver supports. The "features" indicates the feature
  50          * bits after the rte_vhost_driver_features_disable/enable().
  51          * It is also the final feature bits used for vhost-user
  52          * features negotiation.
  53          */
  54         uint64_t supported_features;
  55         uint64_t features;
  56
  57         uint64_t protocol_features;
  58
  59         struct rte_vdpa_device *vdpa_dev;
  60
  61         struct vhost_device_ops const *notify_ops;
  62 };
  63
  64 struct vhost_user_connection {
  65         struct vhost_user_socket *vsocket;
  66         int connfd;
  67         int vid;
  68
  69         TAILQ_ENTRY(vhost_user_connection) next;
  70 };
  71
  72 #define MAX_VHOST_SOCKET 1024
  73 struct vhost_user {
  74         struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET];
  75         struct fdset fdset;
  76         int vsocket_cnt;
  77         pthread_mutex_t mutex;
  78 };
  79
  80 #define MAX_VIRTIO_BACKLOG 128
  81
  82 static void vhost_user_server_new_connection(int fd, void *data, int *remove);
  83 static void vhost_user_read_cb(int fd, void *dat, int *remove);
  84 static int create_unix_socket(struct vhost_user_socket *vsocket);
  85 static int vhost_user_start_client(struct vhost_user_socket *vsocket);
  86
  87 static struct vhost_user vhost_user = {
  88         .fdset = {
  89                 .fd = { [0 ... MAX_FDS - 1] = {-1, NULL, NULL, NULL, 0} },
  90                 .fd_mutex = PTHREAD_MUTEX_INITIALIZER,
  91                 .fd_pooling_mutex = PTHREAD_MUTEX_INITIALIZER,
  92                 .num = 0
  93         },
  94         .vsocket_cnt = 0,
  95         .mutex = PTHREAD_MUTEX_INITIALIZER,
  96 };
  97
  98 /*
  99  * return bytes# of read on success or negative val on failure. Update fdnum
 100  * with number of fds read.
 101  */
 102 int
 103 read_fd_message(int sockfd, char *buf, int buflen, int *fds, int max_fds,
 104                 int *fd_num)
 105 {
 106         struct iovec iov;
 107         struct msghdr msgh;
 108         char control[CMSG_SPACE(max_fds * sizeof(int))];
 109         struct cmsghdr *cmsg;
 110         int got_fds = 0;
 111         int ret;
 112
 113         *fd_num = 0;
 114
 115         memset(&msgh, 0, sizeof(msgh));
 116         iov.iov_base = buf;
 117         iov.iov_len  = buflen;
 118
 119         msgh.msg_iov = &iov;
 120         msgh.msg_iovlen = 1;
 121         msgh.msg_control = control;
 122         msgh.msg_controllen = sizeof(control);
 123
 124         ret = recvmsg(sockfd, &msgh, 0);
 125         if (ret <= 0) {
 126                 if (ret)
 127                         VHOST_LOG_CONFIG(ERR, "recvmsg failed\n");
 128                 return ret;
 129         }
 130
 131         if (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) {
 132                 VHOST_LOG_CONFIG(ERR, "truncated msg\n");
 133                 return -1;
 134         }
 135
 136         for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL;
 137                 cmsg = CMSG_NXTHDR(&msgh, cmsg)) {
 138                 if ((cmsg->cmsg_level == SOL_SOCKET) &&
 139                         (cmsg->cmsg_type == SCM_RIGHTS)) {
 140                         got_fds = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int);
 141                         *fd_num = got_fds;
 142                         memcpy(fds, CMSG_DATA(cmsg), got_fds * sizeof(int));
 143                         break;
 144                 }
 145         }
 146
 147         /* Clear out unused file descriptors */
 148         while (got_fds < max_fds)
 149                 fds[got_fds++] = -1;
 150
 151         return ret;
 152 }
 153
 154 int
 155 send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num)
 156 {
 157
 158         struct iovec iov;
 159         struct msghdr msgh;
 160         size_t fdsize = fd_num * sizeof(int);
 161         char control[CMSG_SPACE(fdsize)];
 162         struct cmsghdr *cmsg;
 163         int ret;
 164
 165         memset(&msgh, 0, sizeof(msgh));
 166         iov.iov_base = buf;
 167         iov.iov_len = buflen;
 168
 169         msgh.msg_iov = &iov;
 170         msgh.msg_iovlen = 1;
 171
 172         if (fds && fd_num > 0) {
 173                 msgh.msg_control = control;
 174                 msgh.msg_controllen = sizeof(control);
 175                 cmsg = CMSG_FIRSTHDR(&msgh);
 176                 if (cmsg == NULL) {
 177                         VHOST_LOG_CONFIG(ERR, "cmsg == NULL\n");
 178                         errno = EINVAL;
 179                         return -1;
 180                 }
 181                 cmsg->cmsg_len = CMSG_LEN(fdsize);
 182                 cmsg->cmsg_level = SOL_SOCKET;
 183                 cmsg->cmsg_type = SCM_RIGHTS;
 184                 memcpy(CMSG_DATA(cmsg), fds, fdsize);
 185         } else {
 186                 msgh.msg_control = NULL;
 187                 msgh.msg_controllen = 0;
 188         }
 189
 190         do {
 191                 ret = sendmsg(sockfd, &msgh, MSG_NOSIGNAL);
 192         } while (ret < 0 && errno == EINTR);
 193
 194         if (ret < 0) {
 195                 VHOST_LOG_CONFIG(ERR,  "sendmsg error\n");
 196                 return ret;
 197         }
 198
 199         return ret;
 200 }
 201
 202 static void
 203 vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
 204 {
 205         int vid;
 206         size_t size;
 207         struct vhost_user_connection *conn;
 208         int ret;
 209         struct virtio_net *dev;
 210
 211         if (vsocket == NULL)
 212                 return;
 213
 214         conn = malloc(sizeof(*conn));
 215         if (conn == NULL) {
 216                 close(fd);
 217                 return;
 218         }
 219
 220         vid = vhost_new_device();
 221         if (vid == -1) {
 222                 goto err;
 223         }
 224
 225         size = strnlen(vsocket->path, PATH_MAX);
 226         vhost_set_ifname(vid, vsocket->path, size);
 227
 228         vhost_set_builtin_virtio_net(vid, vsocket->use_builtin_virtio_net);
 229
 230         vhost_attach_vdpa_device(vid, vsocket->vdpa_dev);
 231
 232         if (vsocket->dequeue_zero_copy)
 233                 vhost_enable_dequeue_zero_copy(vid);
 234
 235         if (vsocket->extbuf)
 236                 vhost_enable_extbuf(vid);
 237
 238         if (vsocket->linearbuf)
 239                 vhost_enable_linearbuf(vid);
 240
 241         if (vsocket->async_copy) {
 242                 dev = get_device(vid);
 243
 244                 if (dev)
 245                         dev->async_copy = 1;
 246         }
 247
 248         VHOST_LOG_CONFIG(INFO, "new device, handle is %d\n", vid);
 249
 250         if (vsocket->notify_ops->new_connection) {
 251                 ret = vsocket->notify_ops->new_connection(vid);
 252                 if (ret < 0) {
 253                         VHOST_LOG_CONFIG(ERR,
 254                                 "failed to add vhost user connection with fd %d\n",
 255                                 fd);
 256                         goto err_cleanup;
 257                 }
 258         }
 259
 260         conn->connfd = fd;
 261         conn->vsocket = vsocket;
 262         conn->vid = vid;
 263         ret = fdset_add(&vhost_user.fdset, fd, vhost_user_read_cb,
 264                         NULL, conn);
 265         if (ret < 0) {
 266                 VHOST_LOG_CONFIG(ERR,
 267                         "failed to add fd %d into vhost server fdset\n",
 268                         fd);
 269
 270                 if (vsocket->notify_ops->destroy_connection)
 271                         vsocket->notify_ops->destroy_connection(conn->vid);
 272
 273                 goto err_cleanup;
 274         }
 275
 276         pthread_mutex_lock(&vsocket->conn_mutex);
 277         TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next);
 278         pthread_mutex_unlock(&vsocket->conn_mutex);
 279
 280         fdset_pipe_notify(&vhost_user.fdset);
 281         return;
 282
 283 err_cleanup:
 284         vhost_destroy_device(vid);
 285 err:
 286         free(conn);
 287         close(fd);
 288 }
 289
 290 /* call back when there is new vhost-user connection from client  */
 291 static void
 292 vhost_user_server_new_connection(int fd, void *dat, int *remove __rte_unused)
 293 {
 294         struct vhost_user_socket *vsocket = dat;
 295
 296         fd = accept(fd, NULL, NULL);
 297         if (fd < 0)
 298                 return;
 299
 300         VHOST_LOG_CONFIG(INFO, "new vhost user connection is %d\n", fd);
 301         vhost_user_add_connection(fd, vsocket);
 302 }
 303
 304 static void
 305 vhost_user_read_cb(int connfd, void *dat, int *remove)
 306 {
 307         struct vhost_user_connection *conn = dat;
 308         struct vhost_user_socket *vsocket = conn->vsocket;
 309         int ret;
 310
 311         ret = vhost_user_msg_handler(conn->vid, connfd);
 312         if (ret < 0) {
 313                 struct virtio_net *dev = get_device(conn->vid);
 314
 315                 close(connfd);
 316                 *remove = 1;
 317
 318                 if (dev)
 319                         vhost_destroy_device_notify(dev);
 320
 321                 if (vsocket->notify_ops->destroy_connection)
 322                         vsocket->notify_ops->destroy_connection(conn->vid);
 323
 324                 vhost_destroy_device(conn->vid);
 325
 326                 if (vsocket->reconnect) {
 327                         create_unix_socket(vsocket);
 328                         vhost_user_start_client(vsocket);
 329                 }
 330
 331                 pthread_mutex_lock(&vsocket->conn_mutex);
 332                 TAILQ_REMOVE(&vsocket->conn_list, conn, next);
 333                 pthread_mutex_unlock(&vsocket->conn_mutex);
 334
 335                 free(conn);
 336         }
 337 }
 338
 339 static int
 340 create_unix_socket(struct vhost_user_socket *vsocket)
 341 {
 342         int fd;
 343         struct sockaddr_un *un = &vsocket->un;
 344
 345         fd = socket(AF_UNIX, SOCK_STREAM, 0);
 346         if (fd < 0)
 347                 return -1;
 348         VHOST_LOG_CONFIG(INFO, "vhost-user %s: socket created, fd: %d\n",
 349                 vsocket->is_server ? "server" : "client", fd);
 350
 351         if (!vsocket->is_server && fcntl(fd, F_SETFL, O_NONBLOCK)) {
 352                 VHOST_LOG_CONFIG(ERR,
 353                         "vhost-user: can't set nonblocking mode for socket, fd: "
 354                         "%d (%s)\n", fd, strerror(errno));
 355                 close(fd);
 356                 return -1;
 357         }
 358
 359         memset(un, 0, sizeof(*un));
 360         un->sun_family = AF_UNIX;
 361         strncpy(un->sun_path, vsocket->path, sizeof(un->sun_path));
 362         un->sun_path[sizeof(un->sun_path) - 1] = '\0';
 363
 364         vsocket->socket_fd = fd;
 365         return 0;
 366 }
 367
 368 static int
 369 vhost_user_start_server(struct vhost_user_socket *vsocket)
 370 {
 371         int ret;
 372         int fd = vsocket->socket_fd;
 373         const char *path = vsocket->path;
 374
 375         /*
 376          * bind () may fail if the socket file with the same name already
 377          * exists. But the library obviously should not delete the file
 378          * provided by the user, since we can not be sure that it is not
 379          * being used by other applications. Moreover, many applications form
 380          * socket names based on user input, which is prone to errors.
 381          *
 382          * The user must ensure that the socket does not exist before
 383          * registering the vhost driver in server mode.
 384          */
 385         ret = bind(fd, (struct sockaddr *)&vsocket->un, sizeof(vsocket->un));
 386         if (ret < 0) {
 387                 VHOST_LOG_CONFIG(ERR,
 388                         "failed to bind to %s: %s; remove it and try again\n",
 389                         path, strerror(errno));
 390                 goto err;
 391         }
 392         VHOST_LOG_CONFIG(INFO, "bind to %s\n", path);
 393
 394         ret = listen(fd, MAX_VIRTIO_BACKLOG);
 395         if (ret < 0)
 396                 goto err;
 397
 398         ret = fdset_add(&vhost_user.fdset, fd, vhost_user_server_new_connection,
 399                   NULL, vsocket);
 400         if (ret < 0) {
 401                 VHOST_LOG_CONFIG(ERR,
 402                         "failed to add listen fd %d to vhost server fdset\n",
 403                         fd);
 404                 goto err;
 405         }
 406
 407         return 0;
 408
 409 err:
 410         close(fd);
 411         return -1;
 412 }
 413
 414 struct vhost_user_reconnect {
 415         struct sockaddr_un un;
 416         int fd;
 417         struct vhost_user_socket *vsocket;
 418
 419         TAILQ_ENTRY(vhost_user_reconnect) next;
 420 };
 421
 422 TAILQ_HEAD(vhost_user_reconnect_tailq_list, vhost_user_reconnect);
 423 struct vhost_user_reconnect_list {
 424         struct vhost_user_reconnect_tailq_list head;
 425         pthread_mutex_t mutex;
 426 };
 427
 428 static struct vhost_user_reconnect_list reconn_list;
 429 static pthread_t reconn_tid;
 430
 431 static int
 432 vhost_user_connect_nonblock(int fd, struct sockaddr *un, size_t sz)
 433 {
 434         int ret, flags;
 435
 436         ret = connect(fd, un, sz);
 437         if (ret < 0 && errno != EISCONN)
 438                 return -1;
 439
 440         flags = fcntl(fd, F_GETFL, 0);
 441         if (flags < 0) {
 442                 VHOST_LOG_CONFIG(ERR,
 443                         "can't get flags for connfd %d\n", fd);
 444                 return -2;
 445         }
 446         if ((flags & O_NONBLOCK) && fcntl(fd, F_SETFL, flags & ~O_NONBLOCK)) {
 447                 VHOST_LOG_CONFIG(ERR,
 448                                 "can't disable nonblocking on fd %d\n", fd);
 449                 return -2;
 450         }
 451         return 0;
 452 }
 453
 454 static void *
 455 vhost_user_client_reconnect(void *arg __rte_unused)
 456 {
 457         int ret;
 458         struct vhost_user_reconnect *reconn, *next;
 459
 460         while (1) {
 461                 pthread_mutex_lock(&reconn_list.mutex);
 462
 463                 /*
 464                  * An equal implementation of TAILQ_FOREACH_SAFE,
 465                  * which does not exist on all platforms.
 466                  */
 467                 for (reconn = TAILQ_FIRST(&reconn_list.head);
 468                      reconn != NULL; reconn = next) {
 469                         next = TAILQ_NEXT(reconn, next);
 470
 471                         ret = vhost_user_connect_nonblock(reconn->fd,
 472                                                 (struct sockaddr *)&reconn->un,
 473                                                 sizeof(reconn->un));
 474                         if (ret == -2) {
 475                                 close(reconn->fd);
 476                                 VHOST_LOG_CONFIG(ERR,
 477                                         "reconnection for fd %d failed\n",
 478                                         reconn->fd);
 479                                 goto remove_fd;
 480                         }
 481                         if (ret == -1)
 482                                 continue;
 483
 484                         VHOST_LOG_CONFIG(INFO,
 485                                 "%s: connected\n", reconn->vsocket->path);
 486                         vhost_user_add_connection(reconn->fd, reconn->vsocket);
 487 remove_fd:
 488                         TAILQ_REMOVE(&reconn_list.head, reconn, next);
 489                         free(reconn);
 490                 }
 491
 492                 pthread_mutex_unlock(&reconn_list.mutex);
 493                 sleep(1);
 494         }
 495
 496         return NULL;
 497 }
 498
 499 static int
 500 vhost_user_reconnect_init(void)
 501 {
 502         int ret;
 503
 504         ret = pthread_mutex_init(&reconn_list.mutex, NULL);
 505         if (ret < 0) {
 506                 VHOST_LOG_CONFIG(ERR, "failed to initialize mutex");
 507                 return ret;
 508         }
 509         TAILQ_INIT(&reconn_list.head);
 510
 511         ret = rte_ctrl_thread_create(&reconn_tid, "vhost_reconn", NULL,
 512                              vhost_user_client_reconnect, NULL);
 513         if (ret != 0) {
 514                 VHOST_LOG_CONFIG(ERR, "failed to create reconnect thread");
 515                 if (pthread_mutex_destroy(&reconn_list.mutex)) {
 516                         VHOST_LOG_CONFIG(ERR,
 517                                 "failed to destroy reconnect mutex");
 518                 }
 519         }
 520
 521         return ret;
 522 }
 523
 524 static int
 525 vhost_user_start_client(struct vhost_user_socket *vsocket)
 526 {
 527         int ret;
 528         int fd = vsocket->socket_fd;
 529         const char *path = vsocket->path;
 530         struct vhost_user_reconnect *reconn;
 531
 532         ret = vhost_user_connect_nonblock(fd, (struct sockaddr *)&vsocket->un,
 533                                           sizeof(vsocket->un));
 534         if (ret == 0) {
 535                 vhost_user_add_connection(fd, vsocket);
 536                 return 0;
 537         }
 538
 539         VHOST_LOG_CONFIG(WARNING,
 540                 "failed to connect to %s: %s\n",
 541                 path, strerror(errno));
 542
 543         if (ret == -2 || !vsocket->reconnect) {
 544                 close(fd);
 545                 return -1;
 546         }
 547
 548         VHOST_LOG_CONFIG(INFO, "%s: reconnecting...\n", path);
 549         reconn = malloc(sizeof(*reconn));
 550         if (reconn == NULL) {
 551                 VHOST_LOG_CONFIG(ERR,
 552                         "failed to allocate memory for reconnect\n");
 553                 close(fd);
 554                 return -1;
 555         }
 556         reconn->un = vsocket->un;
 557         reconn->fd = fd;
 558         reconn->vsocket = vsocket;
 559         pthread_mutex_lock(&reconn_list.mutex);
 560         TAILQ_INSERT_TAIL(&reconn_list.head, reconn, next);
 561         pthread_mutex_unlock(&reconn_list.mutex);
 562
 563         return 0;
 564 }
 565
 566 static struct vhost_user_socket *
 567 find_vhost_user_socket(const char *path)
 568 {
 569         int i;
 570
 571         if (path == NULL)
 572                 return NULL;
 573
 574         for (i = 0; i < vhost_user.vsocket_cnt; i++) {
 575                 struct vhost_user_socket *vsocket = vhost_user.vsockets[i];
 576
 577                 if (!strcmp(vsocket->path, path))
 578                         return vsocket;
 579         }
 580
 581         return NULL;
 582 }
 583
 584 int
 585 rte_vhost_driver_attach_vdpa_device(const char *path,
 586                 struct rte_vdpa_device *dev)
 587 {
 588         struct vhost_user_socket *vsocket;
 589
 590         if (dev == NULL || path == NULL)
 591                 return -1;
 592
 593         pthread_mutex_lock(&vhost_user.mutex);
 594         vsocket = find_vhost_user_socket(path);
 595         if (vsocket)
 596                 vsocket->vdpa_dev = dev;
 597         pthread_mutex_unlock(&vhost_user.mutex);
 598
 599         return vsocket ? 0 : -1;
 600 }
 601
 602 int
 603 rte_vhost_driver_detach_vdpa_device(const char *path)
 604 {
 605         struct vhost_user_socket *vsocket;
 606
 607         pthread_mutex_lock(&vhost_user.mutex);
 608         vsocket = find_vhost_user_socket(path);
 609         if (vsocket)
 610                 vsocket->vdpa_dev = NULL;
 611         pthread_mutex_unlock(&vhost_user.mutex);
 612
 613         return vsocket ? 0 : -1;
 614 }
 615
 616 struct rte_vdpa_device *
 617 rte_vhost_driver_get_vdpa_device(const char *path)
 618 {
 619         struct vhost_user_socket *vsocket;
 620         struct rte_vdpa_device *dev = NULL;
 621
 622         pthread_mutex_lock(&vhost_user.mutex);
 623         vsocket = find_vhost_user_socket(path);
 624         if (vsocket)
 625                 dev = vsocket->vdpa_dev;
 626         pthread_mutex_unlock(&vhost_user.mutex);
 627
 628         return dev;
 629 }
 630
 631 int
 632 rte_vhost_driver_disable_features(const char *path, uint64_t features)
 633 {
 634         struct vhost_user_socket *vsocket;
 635
 636         pthread_mutex_lock(&vhost_user.mutex);
 637         vsocket = find_vhost_user_socket(path);
 638
 639         /* Note that use_builtin_virtio_net is not affected by this function
 640          * since callers may want to selectively disable features of the
 641          * built-in vhost net device backend.
 642          */
 643
 644         if (vsocket)
 645                 vsocket->features &= ~features;
 646         pthread_mutex_unlock(&vhost_user.mutex);
 647
 648         return vsocket ? 0 : -1;
 649 }
 650
 651 int
 652 rte_vhost_driver_enable_features(const char *path, uint64_t features)
 653 {
 654         struct vhost_user_socket *vsocket;
 655
 656         pthread_mutex_lock(&vhost_user.mutex);
 657         vsocket = find_vhost_user_socket(path);
 658         if (vsocket) {
 659                 if ((vsocket->supported_features & features) != features) {
 660                         /*
 661                          * trying to enable features the driver doesn't
 662                          * support.
 663                          */
 664                         pthread_mutex_unlock(&vhost_user.mutex);
 665                         return -1;
 666                 }
 667                 vsocket->features |= features;
 668         }
 669         pthread_mutex_unlock(&vhost_user.mutex);
 670
 671         return vsocket ? 0 : -1;
 672 }
 673
 674 int
 675 rte_vhost_driver_set_features(const char *path, uint64_t features)
 676 {
 677         struct vhost_user_socket *vsocket;
 678
 679         pthread_mutex_lock(&vhost_user.mutex);
 680         vsocket = find_vhost_user_socket(path);
 681         if (vsocket) {
 682                 vsocket->supported_features = features;
 683                 vsocket->features = features;
 684
 685                 /* Anyone setting feature bits is implementing their own vhost
 686                  * device backend.
 687                  */
 688                 vsocket->use_builtin_virtio_net = false;
 689         }
 690         pthread_mutex_unlock(&vhost_user.mutex);
 691
 692         return vsocket ? 0 : -1;
 693 }
 694
 695 int
 696 rte_vhost_driver_get_features(const char *path, uint64_t *features)
 697 {
 698         struct vhost_user_socket *vsocket;
 699         uint64_t vdpa_features;
 700         struct rte_vdpa_device *vdpa_dev;
 701         int ret = 0;
 702
 703         pthread_mutex_lock(&vhost_user.mutex);
 704         vsocket = find_vhost_user_socket(path);
 705         if (!vsocket) {
 706                 VHOST_LOG_CONFIG(ERR,
 707                         "socket file %s is not registered yet.\n", path);
 708                 ret = -1;
 709                 goto unlock_exit;
 710         }
 711
 712         vdpa_dev = vsocket->vdpa_dev;
 713         if (!vdpa_dev) {
 714                 *features = vsocket->features;
 715                 goto unlock_exit;
 716         }
 717
 718         if (vdpa_dev->ops->get_features(vdpa_dev, &vdpa_features) < 0) {
 719                 VHOST_LOG_CONFIG(ERR,
 720                                 "failed to get vdpa features "
 721                                 "for socket file %s.\n", path);
 722                 ret = -1;
 723                 goto unlock_exit;
 724         }
 725
 726         *features = vsocket->features & vdpa_features;
 727
 728 unlock_exit:
 729         pthread_mutex_unlock(&vhost_user.mutex);
 730         return ret;
 731 }
 732
 733 int
 734 rte_vhost_driver_set_protocol_features(const char *path,
 735                 uint64_t protocol_features)
 736 {
 737         struct vhost_user_socket *vsocket;
 738
 739         pthread_mutex_lock(&vhost_user.mutex);
 740         vsocket = find_vhost_user_socket(path);
 741         if (vsocket)
 742                 vsocket->protocol_features = protocol_features;
 743         pthread_mutex_unlock(&vhost_user.mutex);
 744         return vsocket ? 0 : -1;
 745 }
 746
 747 int
 748 rte_vhost_driver_get_protocol_features(const char *path,
 749                 uint64_t *protocol_features)
 750 {
 751         struct vhost_user_socket *vsocket;
 752         uint64_t vdpa_protocol_features;
 753         struct rte_vdpa_device *vdpa_dev;
 754         int ret = 0;
 755
 756         pthread_mutex_lock(&vhost_user.mutex);
 757         vsocket = find_vhost_user_socket(path);
 758         if (!vsocket) {
 759                 VHOST_LOG_CONFIG(ERR,
 760                         "socket file %s is not registered yet.\n", path);
 761                 ret = -1;
 762                 goto unlock_exit;
 763         }
 764
 765         vdpa_dev = vsocket->vdpa_dev;
 766         if (!vdpa_dev) {
 767                 *protocol_features = vsocket->protocol_features;
 768                 goto unlock_exit;
 769         }
 770
 771         if (vdpa_dev->ops->get_protocol_features(vdpa_dev,
 772                                 &vdpa_protocol_features) < 0) {
 773                 VHOST_LOG_CONFIG(ERR,
 774                                 "failed to get vdpa protocol features "
 775                                 "for socket file %s.\n", path);
 776                 ret = -1;
 777                 goto unlock_exit;
 778         }
 779
 780         *protocol_features = vsocket->protocol_features
 781                 & vdpa_protocol_features;
 782
 783 unlock_exit:
 784         pthread_mutex_unlock(&vhost_user.mutex);
 785         return ret;
 786 }
 787
 788 int
 789 rte_vhost_driver_get_queue_num(const char *path, uint32_t *queue_num)
 790 {
 791         struct vhost_user_socket *vsocket;
 792         uint32_t vdpa_queue_num;
 793         struct rte_vdpa_device *vdpa_dev;
 794         int ret = 0;
 795
 796         pthread_mutex_lock(&vhost_user.mutex);
 797         vsocket = find_vhost_user_socket(path);
 798         if (!vsocket) {
 799                 VHOST_LOG_CONFIG(ERR,
 800                         "socket file %s is not registered yet.\n", path);
 801                 ret = -1;
 802                 goto unlock_exit;
 803         }
 804
 805         vdpa_dev = vsocket->vdpa_dev;
 806         if (!vdpa_dev) {
 807                 *queue_num = VHOST_MAX_QUEUE_PAIRS;
 808                 goto unlock_exit;
 809         }
 810
 811         if (vdpa_dev->ops->get_queue_num(vdpa_dev, &vdpa_queue_num) < 0) {
 812                 VHOST_LOG_CONFIG(ERR,
 813                                 "failed to get vdpa queue number "
 814                                 "for socket file %s.\n", path);
 815                 ret = -1;
 816                 goto unlock_exit;
 817         }
 818
 819         *queue_num = RTE_MIN((uint32_t)VHOST_MAX_QUEUE_PAIRS, vdpa_queue_num);
 820
 821 unlock_exit:
 822         pthread_mutex_unlock(&vhost_user.mutex);
 823         return ret;
 824 }
 825
 826 static void
 827 vhost_user_socket_mem_free(struct vhost_user_socket *vsocket)
 828 {
 829         if (vsocket && vsocket->path) {
 830                 free(vsocket->path);
 831                 vsocket->path = NULL;
 832         }
 833
 834         if (vsocket) {
 835                 free(vsocket);
 836                 vsocket = NULL;
 837         }
 838 }
 839
 840 /*
 841  * Register a new vhost-user socket; here we could act as server
 842  * (the default case), or client (when RTE_VHOST_USER_CLIENT) flag
 843  * is set.
 844  */
 845 int
 846 rte_vhost_driver_register(const char *path, uint64_t flags)
 847 {
 848         int ret = -1;
 849         struct vhost_user_socket *vsocket;
 850
 851         if (!path)
 852                 return -1;
 853
 854         pthread_mutex_lock(&vhost_user.mutex);
 855
 856         if (vhost_user.vsocket_cnt == MAX_VHOST_SOCKET) {
 857                 VHOST_LOG_CONFIG(ERR,
 858                         "error: the number of vhost sockets reaches maximum\n");
 859                 goto out;
 860         }
 861
 862         vsocket = malloc(sizeof(struct vhost_user_socket));
 863         if (!vsocket)
 864                 goto out;
 865         memset(vsocket, 0, sizeof(struct vhost_user_socket));
 866         vsocket->path = strdup(path);
 867         if (vsocket->path == NULL) {
 868                 VHOST_LOG_CONFIG(ERR,
 869                         "error: failed to copy socket path string\n");
 870                 vhost_user_socket_mem_free(vsocket);
 871                 goto out;
 872         }
 873         TAILQ_INIT(&vsocket->conn_list);
 874         ret = pthread_mutex_init(&vsocket->conn_mutex, NULL);
 875         if (ret) {
 876                 VHOST_LOG_CONFIG(ERR,
 877                         "error: failed to init connection mutex\n");
 878                 goto out_free;
 879         }
 880         vsocket->vdpa_dev = NULL;
 881         vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
 882         vsocket->extbuf = flags & RTE_VHOST_USER_EXTBUF_SUPPORT;
 883         vsocket->linearbuf = flags & RTE_VHOST_USER_LINEARBUF_SUPPORT;
 884
 885         if (vsocket->dequeue_zero_copy &&
 886             (flags & RTE_VHOST_USER_IOMMU_SUPPORT)) {
 887                 VHOST_LOG_CONFIG(ERR,
 888                         "error: enabling dequeue zero copy and IOMMU features "
 889                         "simultaneously is not supported\n");
 890                 goto out_mutex;
 891         }
 892
 893         vsocket->async_copy = flags & RTE_VHOST_USER_ASYNC_COPY;
 894
 895         if (vsocket->async_copy &&
 896                 (flags & (RTE_VHOST_USER_IOMMU_SUPPORT |
 897                 RTE_VHOST_USER_POSTCOPY_SUPPORT))) {
 898                 VHOST_LOG_CONFIG(ERR, "error: enabling async copy and IOMMU "
 899                         "or post-copy feature simultaneously is not "
 900                         "supported\n");
 901                 goto out_mutex;
 902         }
 903
 904         /*
 905          * Set the supported features correctly for the builtin vhost-user
 906          * net driver.
 907          *
 908          * Applications know nothing about features the builtin virtio net
 909          * driver (virtio_net.c) supports, thus it's not possible for them
 910          * to invoke rte_vhost_driver_set_features(). To workaround it, here
 911          * we set it unconditionally. If the application want to implement
 912          * another vhost-user driver (say SCSI), it should call the
 913          * rte_vhost_driver_set_features(), which will overwrite following
 914          * two values.
 915          */
 916         vsocket->use_builtin_virtio_net = true;
 917         vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
 918         vsocket->features           = VIRTIO_NET_SUPPORTED_FEATURES;
 919         vsocket->protocol_features  = VHOST_USER_PROTOCOL_FEATURES;
 920
 921         /*
 922          * Dequeue zero copy can't assure descriptors returned in order.
 923          * Also, it requires that the guest memory is populated, which is
 924          * not compatible with postcopy.
 925          */
 926         if (vsocket->dequeue_zero_copy) {
 927                 if (vsocket->extbuf) {
 928                         VHOST_LOG_CONFIG(ERR,
 929                         "error: zero copy is incompatible with external buffers\n");
 930                         ret = -1;
 931                         goto out_mutex;
 932                 }
 933                 if (vsocket->linearbuf) {
 934                         VHOST_LOG_CONFIG(ERR,
 935                         "error: zero copy is incompatible with linear buffers\n");
 936                         ret = -1;
 937                         goto out_mutex;
 938                 }
 939                 if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
 940                         VHOST_LOG_CONFIG(ERR,
 941                         "error: zero copy is incompatible with vhost client mode\n");
 942                         ret = -1;
 943                         goto out_mutex;
 944                 }
 945                 vsocket->supported_features &= ~(1ULL << VIRTIO_F_IN_ORDER);
 946                 vsocket->features &= ~(1ULL << VIRTIO_F_IN_ORDER);
 947
 948                 VHOST_LOG_CONFIG(INFO,
 949                         "Dequeue zero copy requested, disabling postcopy support\n");
 950                 vsocket->protocol_features &=
 951                         ~(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT);
 952         }
 953
 954         if (vsocket->async_copy) {
 955                 vsocket->supported_features &= ~(1ULL << VHOST_F_LOG_ALL);
 956                 vsocket->features &= ~(1ULL << VHOST_F_LOG_ALL);
 957                 VHOST_LOG_CONFIG(INFO,
 958                         "Logging feature is disabled in async copy mode\n");
 959         }
 960
 961         /*
 962          * We'll not be able to receive a buffer from guest in linear mode
 963          * without external buffer if it will not fit in a single mbuf, which is
 964          * likely if segmentation offloading enabled.
 965          */
 966         if (vsocket->linearbuf && !vsocket->extbuf) {
 967                 uint64_t seg_offload_features =
 968                                 (1ULL << VIRTIO_NET_F_HOST_TSO4) |
 969                                 (1ULL << VIRTIO_NET_F_HOST_TSO6) |
 970                                 (1ULL << VIRTIO_NET_F_HOST_UFO);
 971
 972                 VHOST_LOG_CONFIG(INFO,
 973                         "Linear buffers requested without external buffers, "
 974                         "disabling host segmentation offloading support\n");
 975                 vsocket->supported_features &= ~seg_offload_features;
 976                 vsocket->features &= ~seg_offload_features;
 977         }
 978
 979         if (!(flags & RTE_VHOST_USER_IOMMU_SUPPORT)) {
 980                 vsocket->supported_features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 981                 vsocket->features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 982         }
 983
 984         if (!(flags & RTE_VHOST_USER_POSTCOPY_SUPPORT)) {
 985                 vsocket->protocol_features &=
 986                         ~(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT);
 987         } else {
 988 #ifndef RTE_LIBRTE_VHOST_POSTCOPY
 989                 VHOST_LOG_CONFIG(ERR,
 990                         "Postcopy requested but not compiled\n");
 991                 ret = -1;
 992                 goto out_mutex;
 993 #endif
 994         }
 995
 996         if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
 997                 vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
 998                 if (vsocket->reconnect && reconn_tid == 0) {
 999                         if (vhost_user_reconnect_init() != 0)
1000                                 goto out_mutex;
1001                 }
1002         } else {
1003                 vsocket->is_server = true;
1004         }
1005         ret = create_unix_socket(vsocket);
1006         if (ret < 0) {
1007                 goto out_mutex;
1008         }
1009
1010         vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket;
1011
1012         pthread_mutex_unlock(&vhost_user.mutex);
1013         return ret;
1014
1015 out_mutex:
1016         if (pthread_mutex_destroy(&vsocket->conn_mutex)) {
1017                 VHOST_LOG_CONFIG(ERR,
1018                         "error: failed to destroy connection mutex\n");
1019         }
1020 out_free:
1021         vhost_user_socket_mem_free(vsocket);
1022 out:
1023         pthread_mutex_unlock(&vhost_user.mutex);
1024
1025         return ret;
1026 }
1027
1028 static bool
1029 vhost_user_remove_reconnect(struct vhost_user_socket *vsocket)
1030 {
1031         int found = false;
1032         struct vhost_user_reconnect *reconn, *next;
1033
1034         pthread_mutex_lock(&reconn_list.mutex);
1035
1036         for (reconn = TAILQ_FIRST(&reconn_list.head);
1037              reconn != NULL; reconn = next) {
1038                 next = TAILQ_NEXT(reconn, next);
1039
1040                 if (reconn->vsocket == vsocket) {
1041                         TAILQ_REMOVE(&reconn_list.head, reconn, next);
1042                         close(reconn->fd);
1043                         free(reconn);
1044                         found = true;
1045                         break;
1046                 }
1047         }
1048         pthread_mutex_unlock(&reconn_list.mutex);
1049         return found;
1050 }
1051
1052 /**
1053  * Unregister the specified vhost socket
1054  */
1055 int
1056 rte_vhost_driver_unregister(const char *path)
1057 {
1058         int i;
1059         int count;
1060         struct vhost_user_connection *conn, *next;
1061
1062         if (path == NULL)
1063                 return -1;
1064
1065 again:
1066         pthread_mutex_lock(&vhost_user.mutex);
1067
1068         for (i = 0; i < vhost_user.vsocket_cnt; i++) {
1069                 struct vhost_user_socket *vsocket = vhost_user.vsockets[i];
1070
1071                 if (!strcmp(vsocket->path, path)) {
1072                         pthread_mutex_lock(&vsocket->conn_mutex);
1073                         for (conn = TAILQ_FIRST(&vsocket->conn_list);
1074                              conn != NULL;
1075                              conn = next) {
1076                                 next = TAILQ_NEXT(conn, next);
1077
1078                                 /*
1079                                  * If r/wcb is executing, release vsocket's
1080                                  * conn_mutex and vhost_user's mutex locks, and
1081                                  * try again since the r/wcb may use the
1082                                  * conn_mutex and mutex locks.
1083                                  */
1084                                 if (fdset_try_del(&vhost_user.fdset,
1085                                                   conn->connfd) == -1) {
1086                                         pthread_mutex_unlock(
1087                                                         &vsocket->conn_mutex);
1088                                         pthread_mutex_unlock(&vhost_user.mutex);
1089                                         goto again;
1090                                 }
1091
1092                                 VHOST_LOG_CONFIG(INFO,
1093                                         "free connfd = %d for device '%s'\n",
1094                                         conn->connfd, path);
1095                                 close(conn->connfd);
1096                                 vhost_destroy_device(conn->vid);
1097                                 TAILQ_REMOVE(&vsocket->conn_list, conn, next);
1098                                 free(conn);
1099                         }
1100                         pthread_mutex_unlock(&vsocket->conn_mutex);
1101
1102                         if (vsocket->is_server) {
1103                                 /*
1104                                  * If r/wcb is executing, release vhost_user's
1105                                  * mutex lock, and try again since the r/wcb
1106                                  * may use the mutex lock.
1107                                  */
1108                                 if (fdset_try_del(&vhost_user.fdset,
1109                                                 vsocket->socket_fd) == -1) {
1110                                         pthread_mutex_unlock(&vhost_user.mutex);
1111                                         goto again;
1112                                 }
1113
1114                                 close(vsocket->socket_fd);
1115                                 unlink(path);
1116                         } else if (vsocket->reconnect) {
1117                                 vhost_user_remove_reconnect(vsocket);
1118                         }
1119
1120                         pthread_mutex_destroy(&vsocket->conn_mutex);
1121                         vhost_user_socket_mem_free(vsocket);
1122
1123                         count = --vhost_user.vsocket_cnt;
1124                         vhost_user.vsockets[i] = vhost_user.vsockets[count];
1125                         vhost_user.vsockets[count] = NULL;
1126                         pthread_mutex_unlock(&vhost_user.mutex);
1127
1128                         return 0;
1129                 }
1130         }
1131         pthread_mutex_unlock(&vhost_user.mutex);
1132
1133         return -1;
1134 }
1135
1136 /*
1137  * Register ops so that we can add/remove device to data core.
1138  */
1139 int
1140 rte_vhost_driver_callback_register(const char *path,
1141         struct vhost_device_ops const * const ops)
1142 {
1143         struct vhost_user_socket *vsocket;
1144
1145         pthread_mutex_lock(&vhost_user.mutex);
1146         vsocket = find_vhost_user_socket(path);
1147         if (vsocket)
1148                 vsocket->notify_ops = ops;
1149         pthread_mutex_unlock(&vhost_user.mutex);
1150
1151         return vsocket ? 0 : -1;
1152 }
1153
1154 struct vhost_device_ops const *
1155 vhost_driver_callback_get(const char *path)
1156 {
1157         struct vhost_user_socket *vsocket;
1158
1159         pthread_mutex_lock(&vhost_user.mutex);
1160         vsocket = find_vhost_user_socket(path);
1161         pthread_mutex_unlock(&vhost_user.mutex);
1162
1163         return vsocket ? vsocket->notify_ops : NULL;
1164 }
1165
1166 int
1167 rte_vhost_driver_start(const char *path)
1168 {
1169         struct vhost_user_socket *vsocket;
1170         static pthread_t fdset_tid;
1171
1172         pthread_mutex_lock(&vhost_user.mutex);
1173         vsocket = find_vhost_user_socket(path);
1174         pthread_mutex_unlock(&vhost_user.mutex);
1175
1176         if (!vsocket)
1177                 return -1;
1178
1179         if (fdset_tid == 0) {
1180                 /**
1181                  * create a pipe which will be waited by poll and notified to
1182                  * rebuild the wait list of poll.
1183                  */
1184                 if (fdset_pipe_init(&vhost_user.fdset) < 0) {
1185                         VHOST_LOG_CONFIG(ERR,
1186                                 "failed to create pipe for vhost fdset\n");
1187                         return -1;
1188                 }
1189
1190                 int ret = rte_ctrl_thread_create(&fdset_tid,
1191                         "vhost-events", NULL, fdset_event_dispatch,
1192                         &vhost_user.fdset);
1193                 if (ret != 0) {
1194                         VHOST_LOG_CONFIG(ERR,
1195                                 "failed to create fdset handling thread");
1196
1197                         fdset_pipe_uninit(&vhost_user.fdset);
1198                         return -1;
1199                 }
1200         }
1201
1202         if (vsocket->is_server)
1203                 return vhost_user_start_server(vsocket);
1204         else
1205                 return vhost_user_start_client(vsocket);
1206 }