drivers/net/virtio/virtio_user/virtio_user_dev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdio.h>
   7 #include <fcntl.h>
   8 #include <string.h>
   9 #include <errno.h>
  10 #include <sys/mman.h>
  11 #include <unistd.h>
  12 #include <sys/eventfd.h>
  13 #include <sys/types.h>
  14 #include <sys/stat.h>
  15
  16 #include <rte_string_fns.h>
  17 #include <rte_eal_memconfig.h>
  18
  19 #include "vhost.h"
  20 #include "virtio_user_dev.h"
  21 #include "../virtio_ethdev.h"
  22
  23 #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb"
  24
  25 static int
  26 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
  27 {
  28         /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come
  29          * firstly because vhost depends on this msg to allocate virtqueue
  30          * pair.
  31          */
  32         struct vhost_vring_file file;
  33
  34         file.index = queue_sel;
  35         file.fd = dev->callfds[queue_sel];
  36         dev->ops->send_request(dev, VHOST_USER_SET_VRING_CALL, &file);
  37
  38         return 0;
  39 }
  40
  41 static int
  42 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
  43 {
  44         struct vhost_vring_file file;
  45         struct vhost_vring_state state;
  46         struct vring *vring = &dev->vrings[queue_sel];
  47         struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel];
  48         struct vhost_vring_addr addr = {
  49                 .index = queue_sel,
  50                 .log_guest_addr = 0,
  51                 .flags = 0, /* disable log */
  52         };
  53
  54         if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
  55                 addr.desc_user_addr =
  56                         (uint64_t)(uintptr_t)pq_vring->desc;
  57                 addr.avail_user_addr =
  58                         (uint64_t)(uintptr_t)pq_vring->driver;
  59                 addr.used_user_addr =
  60                         (uint64_t)(uintptr_t)pq_vring->device;
  61         } else {
  62                 addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc;
  63                 addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail;
  64                 addr.used_user_addr = (uint64_t)(uintptr_t)vring->used;
  65         }
  66
  67         state.index = queue_sel;
  68         state.num = vring->num;
  69         dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);
  70
  71         state.index = queue_sel;
  72         state.num = 0; /* no reservation */
  73         if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
  74                 state.num |= (1 << 15);
  75         dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);
  76
  77         dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr);
  78
  79         /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes
  80          * lastly because vhost depends on this msg to judge if
  81          * virtio is ready.
  82          */
  83         file.index = queue_sel;
  84         file.fd = dev->kickfds[queue_sel];
  85         dev->ops->send_request(dev, VHOST_USER_SET_VRING_KICK, &file);
  86
  87         return 0;
  88 }
  89
  90 static int
  91 virtio_user_queue_setup(struct virtio_user_dev *dev,
  92                         int (*fn)(struct virtio_user_dev *, uint32_t))
  93 {
  94         uint32_t i, queue_sel;
  95
  96         for (i = 0; i < dev->max_queue_pairs; ++i) {
  97                 queue_sel = 2 * i + VTNET_SQ_RQ_QUEUE_IDX;
  98                 if (fn(dev, queue_sel) < 0) {
  99                         PMD_DRV_LOG(INFO, "setup rx vq fails: %u", i);
 100                         return -1;
 101                 }
 102         }
 103         for (i = 0; i < dev->max_queue_pairs; ++i) {
 104                 queue_sel = 2 * i + VTNET_SQ_TQ_QUEUE_IDX;
 105                 if (fn(dev, queue_sel) < 0) {
 106                         PMD_DRV_LOG(INFO, "setup tx vq fails: %u", i);
 107                         return -1;
 108                 }
 109         }
 110
 111         return 0;
 112 }
 113
 114 int
 115 virtio_user_start_device(struct virtio_user_dev *dev)
 116 {
 117         uint64_t features;
 118         int ret;
 119
 120         /*
 121          * XXX workaround!
 122          *
 123          * We need to make sure that the locks will be
 124          * taken in the correct order to avoid deadlocks.
 125          *
 126          * Before releasing this lock, this thread should
 127          * not trigger any memory hotplug events.
 128          *
 129          * This is a temporary workaround, and should be
 130          * replaced when we get proper supports from the
 131          * memory subsystem in the future.
 132          */
 133         rte_mcfg_mem_read_lock();
 134         pthread_mutex_lock(&dev->mutex);
 135
 136         if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER &&
 137                         dev->vhostfd < 0)
 138                 goto error;
 139
 140         /* Step 0: tell vhost to create queues */
 141         if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0)
 142                 goto error;
 143
 144         /* Step 1: negotiate protocol features & set features */
 145         features = dev->features;
 146
 147
 148         /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */
 149         features &= ~(1ull << VIRTIO_NET_F_MAC);
 150         /* Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */
 151         features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ);
 152         features &= ~(1ull << VIRTIO_NET_F_STATUS);
 153         ret = dev->ops->send_request(dev, VHOST_USER_SET_FEATURES, &features);
 154         if (ret < 0)
 155                 goto error;
 156         PMD_DRV_LOG(INFO, "set features: %" PRIx64, features);
 157
 158         /* Step 2: share memory regions */
 159         ret = dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL);
 160         if (ret < 0)
 161                 goto error;
 162
 163         /* Step 3: kick queues */
 164         if (virtio_user_queue_setup(dev, virtio_user_kick_queue) < 0)
 165                 goto error;
 166
 167         /* Step 4: enable queues
 168          * we enable the 1st queue pair by default.
 169          */
 170         dev->ops->enable_qp(dev, 0, 1);
 171
 172         dev->started = true;
 173         pthread_mutex_unlock(&dev->mutex);
 174         rte_mcfg_mem_read_unlock();
 175
 176         return 0;
 177 error:
 178         pthread_mutex_unlock(&dev->mutex);
 179         rte_mcfg_mem_read_unlock();
 180         /* TODO: free resource here or caller to check */
 181         return -1;
 182 }
 183
 184 int virtio_user_stop_device(struct virtio_user_dev *dev)
 185 {
 186         struct vhost_vring_state state;
 187         uint32_t i;
 188         int error = 0;
 189
 190         pthread_mutex_lock(&dev->mutex);
 191         if (!dev->started)
 192                 goto out;
 193
 194         for (i = 0; i < dev->max_queue_pairs; ++i)
 195                 dev->ops->enable_qp(dev, i, 0);
 196
 197         /* Stop the backend. */
 198         for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
 199                 state.index = i;
 200                 if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE,
 201                                            &state) < 0) {
 202                         PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u\n",
 203                                     i);
 204                         error = -1;
 205                         goto out;
 206                 }
 207         }
 208
 209         dev->started = false;
 210 out:
 211         pthread_mutex_unlock(&dev->mutex);
 212
 213         return error;
 214 }
 215
 216 static inline void
 217 parse_mac(struct virtio_user_dev *dev, const char *mac)
 218 {
 219         struct rte_ether_addr tmp;
 220
 221         if (!mac)
 222                 return;
 223
 224         if (rte_ether_unformat_addr(mac, &tmp) == 0) {
 225                 memcpy(dev->mac_addr, &tmp, RTE_ETHER_ADDR_LEN);
 226                 dev->mac_specified = 1;
 227         } else {
 228                 /* ignore the wrong mac, use random mac */
 229                 PMD_DRV_LOG(ERR, "wrong format of mac: %s", mac);
 230         }
 231 }
 232
 233 static int
 234 virtio_user_dev_init_notify(struct virtio_user_dev *dev)
 235 {
 236         uint32_t i, j;
 237         int callfd;
 238         int kickfd;
 239
 240         for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; ++i) {
 241                 if (i >= dev->max_queue_pairs * 2) {
 242                         dev->kickfds[i] = -1;
 243                         dev->callfds[i] = -1;
 244                         continue;
 245                 }
 246
 247                 /* May use invalid flag, but some backend uses kickfd and
 248                  * callfd as criteria to judge if dev is alive. so finally we
 249                  * use real event_fd.
 250                  */
 251                 callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
 252                 if (callfd < 0) {
 253                         PMD_DRV_LOG(ERR, "callfd error, %s", strerror(errno));
 254                         break;
 255                 }
 256                 kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
 257                 if (kickfd < 0) {
 258                         PMD_DRV_LOG(ERR, "kickfd error, %s", strerror(errno));
 259                         break;
 260                 }
 261                 dev->callfds[i] = callfd;
 262                 dev->kickfds[i] = kickfd;
 263         }
 264
 265         if (i < VIRTIO_MAX_VIRTQUEUES) {
 266                 for (j = 0; j <= i; ++j) {
 267                         close(dev->callfds[j]);
 268                         close(dev->kickfds[j]);
 269                 }
 270
 271                 return -1;
 272         }
 273
 274         return 0;
 275 }
 276
 277 static int
 278 virtio_user_fill_intr_handle(struct virtio_user_dev *dev)
 279 {
 280         uint32_t i;
 281         struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
 282
 283         if (!eth_dev->intr_handle) {
 284                 eth_dev->intr_handle = malloc(sizeof(*eth_dev->intr_handle));
 285                 if (!eth_dev->intr_handle) {
 286                         PMD_DRV_LOG(ERR, "fail to allocate intr_handle");
 287                         return -1;
 288                 }
 289                 memset(eth_dev->intr_handle, 0, sizeof(*eth_dev->intr_handle));
 290         }
 291
 292         for (i = 0; i < dev->max_queue_pairs; ++i)
 293                 eth_dev->intr_handle->efds[i] = dev->callfds[i];
 294         eth_dev->intr_handle->nb_efd = dev->max_queue_pairs;
 295         eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1;
 296         eth_dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
 297         /* For virtio vdev, no need to read counter for clean */
 298         eth_dev->intr_handle->efd_counter_size = 0;
 299         eth_dev->intr_handle->fd = -1;
 300         if (dev->vhostfd >= 0)
 301                 eth_dev->intr_handle->fd = dev->vhostfd;
 302         else if (dev->is_server)
 303                 eth_dev->intr_handle->fd = dev->listenfd;
 304
 305         return 0;
 306 }
 307
 308 static void
 309 virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused,
 310                          const void *addr,
 311                          size_t len __rte_unused,
 312                          void *arg)
 313 {
 314         struct virtio_user_dev *dev = arg;
 315         struct rte_memseg_list *msl;
 316         uint16_t i;
 317
 318         /* ignore externally allocated memory */
 319         msl = rte_mem_virt2memseg_list(addr);
 320         if (msl->external)
 321                 return;
 322
 323         pthread_mutex_lock(&dev->mutex);
 324
 325         if (dev->started == false)
 326                 goto exit;
 327
 328         /* Step 1: pause the active queues */
 329         for (i = 0; i < dev->queue_pairs; i++)
 330                 dev->ops->enable_qp(dev, i, 0);
 331
 332         /* Step 2: update memory regions */
 333         dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL);
 334
 335         /* Step 3: resume the active queues */
 336         for (i = 0; i < dev->queue_pairs; i++)
 337                 dev->ops->enable_qp(dev, i, 1);
 338
 339 exit:
 340         pthread_mutex_unlock(&dev->mutex);
 341 }
 342
 343 static int
 344 virtio_user_dev_setup(struct virtio_user_dev *dev)
 345 {
 346         uint32_t q;
 347
 348         dev->vhostfd = -1;
 349         dev->vhostfds = NULL;
 350         dev->tapfds = NULL;
 351
 352         if (dev->is_server) {
 353                 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) {
 354                         PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!");
 355                         return -1;
 356                 }
 357                 dev->ops = &virtio_ops_user;
 358         } else {
 359                 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) {
 360                         dev->ops = &virtio_ops_user;
 361                 } else if (dev->backend_type ==
 362                                         VIRTIO_USER_BACKEND_VHOST_KERNEL) {
 363                         dev->ops = &virtio_ops_kernel;
 364
 365                         dev->vhostfds = malloc(dev->max_queue_pairs *
 366                                                sizeof(int));
 367                         dev->tapfds = malloc(dev->max_queue_pairs *
 368                                              sizeof(int));
 369                         if (!dev->vhostfds || !dev->tapfds) {
 370                                 PMD_INIT_LOG(ERR, "Failed to malloc");
 371                                 return -1;
 372                         }
 373
 374                         for (q = 0; q < dev->max_queue_pairs; ++q) {
 375                                 dev->vhostfds[q] = -1;
 376                                 dev->tapfds[q] = -1;
 377                         }
 378                 }
 379         }
 380
 381         if (dev->ops->setup(dev) < 0)
 382                 return -1;
 383
 384         if (virtio_user_dev_init_notify(dev) < 0)
 385                 return -1;
 386
 387         if (virtio_user_fill_intr_handle(dev) < 0)
 388                 return -1;
 389
 390         return 0;
 391 }
 392
 393 /* Use below macro to filter features from vhost backend */
 394 #define VIRTIO_USER_SUPPORTED_FEATURES                  \
 395         (1ULL << VIRTIO_NET_F_MAC               |       \
 396          1ULL << VIRTIO_NET_F_STATUS            |       \
 397          1ULL << VIRTIO_NET_F_MQ                |       \
 398          1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR     |       \
 399          1ULL << VIRTIO_NET_F_CTRL_VQ           |       \
 400          1ULL << VIRTIO_NET_F_CTRL_RX           |       \
 401          1ULL << VIRTIO_NET_F_CTRL_VLAN         |       \
 402          1ULL << VIRTIO_NET_F_CSUM              |       \
 403          1ULL << VIRTIO_NET_F_HOST_TSO4         |       \
 404          1ULL << VIRTIO_NET_F_HOST_TSO6         |       \
 405          1ULL << VIRTIO_NET_F_MRG_RXBUF         |       \
 406          1ULL << VIRTIO_RING_F_INDIRECT_DESC    |       \
 407          1ULL << VIRTIO_NET_F_GUEST_CSUM        |       \
 408          1ULL << VIRTIO_NET_F_GUEST_TSO4        |       \
 409          1ULL << VIRTIO_NET_F_GUEST_TSO6        |       \
 410          1ULL << VIRTIO_F_IN_ORDER              |       \
 411          1ULL << VIRTIO_F_VERSION_1             |       \
 412          1ULL << VIRTIO_F_RING_PACKED           |       \
 413          1ULL << VHOST_USER_F_PROTOCOL_FEATURES)
 414
 415 #define VIRTIO_USER_SUPPORTED_PROTOCOL_FEATURES         \
 416         (1ULL << VHOST_USER_PROTOCOL_F_MQ |             \
 417          1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK |      \
 418          1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 419
 420 int
 421 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
 422                      int cq, int queue_size, const char *mac, char **ifname,
 423                      int server, int mrg_rxbuf, int in_order, int packed_vq,
 424                      enum virtio_user_backend_type backend_type)
 425 {
 426         uint64_t protocol_features = 0;
 427
 428         pthread_mutex_init(&dev->mutex, NULL);
 429         strlcpy(dev->path, path, PATH_MAX);
 430         dev->started = 0;
 431         dev->max_queue_pairs = queues;
 432         dev->queue_pairs = 1; /* mq disabled by default */
 433         dev->queue_size = queue_size;
 434         dev->is_server = server;
 435         dev->mac_specified = 0;
 436         dev->frontend_features = 0;
 437         dev->unsupported_features = ~VIRTIO_USER_SUPPORTED_FEATURES;
 438         dev->protocol_features = VIRTIO_USER_SUPPORTED_PROTOCOL_FEATURES;
 439         dev->backend_type = backend_type;
 440
 441         parse_mac(dev, mac);
 442
 443         if (*ifname) {
 444                 dev->ifname = *ifname;
 445                 *ifname = NULL;
 446         }
 447
 448         if (virtio_user_dev_setup(dev) < 0) {
 449                 PMD_INIT_LOG(ERR, "backend set up fails");
 450                 return -1;
 451         }
 452
 453         if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER)
 454                 dev->unsupported_features |=
 455                         (1ULL << VHOST_USER_F_PROTOCOL_FEATURES);
 456
 457         if (!dev->is_server) {
 458                 if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER,
 459                                            NULL) < 0) {
 460                         PMD_INIT_LOG(ERR, "set_owner fails: %s",
 461                                      strerror(errno));
 462                         return -1;
 463                 }
 464
 465                 if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES,
 466                                            &dev->device_features) < 0) {
 467                         PMD_INIT_LOG(ERR, "get_features failed: %s",
 468                                      strerror(errno));
 469                         return -1;
 470                 }
 471
 472
 473                 if (dev->device_features &
 474                                 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) {
 475                         if (dev->ops->send_request(dev,
 476                                         VHOST_USER_GET_PROTOCOL_FEATURES,
 477                                         &protocol_features))
 478                                 return -1;
 479
 480                         dev->protocol_features &= protocol_features;
 481
 482                         if (dev->ops->send_request(dev,
 483                                         VHOST_USER_SET_PROTOCOL_FEATURES,
 484                                         &dev->protocol_features))
 485                                 return -1;
 486
 487                         if (!(dev->protocol_features &
 488                                         (1ULL << VHOST_USER_PROTOCOL_F_MQ)))
 489                                 dev->unsupported_features |=
 490                                         (1ull << VIRTIO_NET_F_MQ);
 491                 }
 492         } else {
 493                 /* We just pretend vhost-user can support all these features.
 494                  * Note that this could be problematic that if some feature is
 495                  * negotiated but not supported by the vhost-user which comes
 496                  * later.
 497                  */
 498                 dev->device_features = VIRTIO_USER_SUPPORTED_FEATURES;
 499         }
 500
 501
 502
 503         if (!mrg_rxbuf)
 504                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF);
 505
 506         if (!in_order)
 507                 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER);
 508
 509         if (!packed_vq)
 510                 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED);
 511
 512         if (dev->mac_specified)
 513                 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC);
 514         else
 515                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC);
 516
 517         if (cq) {
 518                 /* device does not really need to know anything about CQ,
 519                  * so if necessary, we just claim to support CQ
 520                  */
 521                 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);
 522         } else {
 523                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);
 524                 /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */
 525                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX);
 526                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN);
 527                 dev->unsupported_features |=
 528                         (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE);
 529                 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);
 530                 dev->unsupported_features |=
 531                         (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR);
 532         }
 533
 534         /* The backend will not report this feature, we add it explicitly */
 535         if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER)
 536                 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS);
 537
 538         /*
 539          * Device features =
 540          *     (frontend_features | backend_features) & ~unsupported_features;
 541          */
 542         dev->device_features |= dev->frontend_features;
 543         dev->device_features &= ~dev->unsupported_features;
 544
 545         if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME,
 546                                 virtio_user_mem_event_cb, dev)) {
 547                 if (rte_errno != ENOTSUP) {
 548                         PMD_INIT_LOG(ERR, "Failed to register mem event"
 549                                         " callback\n");
 550                         return -1;
 551                 }
 552         }
 553
 554         return 0;
 555 }
 556
 557 void
 558 virtio_user_dev_uninit(struct virtio_user_dev *dev)
 559 {
 560         uint32_t i;
 561
 562         virtio_user_stop_device(dev);
 563
 564         rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev);
 565
 566         for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
 567                 close(dev->callfds[i]);
 568                 close(dev->kickfds[i]);
 569         }
 570
 571         if (dev->vhostfd >= 0)
 572                 close(dev->vhostfd);
 573
 574         if (dev->is_server && dev->listenfd >= 0) {
 575                 close(dev->listenfd);
 576                 dev->listenfd = -1;
 577         }
 578
 579         if (dev->vhostfds) {
 580                 for (i = 0; i < dev->max_queue_pairs; ++i) {
 581                         close(dev->vhostfds[i]);
 582                         if (dev->tapfds[i] >= 0)
 583                                 close(dev->tapfds[i]);
 584                 }
 585                 free(dev->vhostfds);
 586                 free(dev->tapfds);
 587         }
 588
 589         free(dev->ifname);
 590
 591         if (dev->is_server)
 592                 unlink(dev->path);
 593 }
 594
 595 uint8_t
 596 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs)
 597 {
 598         uint16_t i;
 599         uint8_t ret = 0;
 600
 601         if (q_pairs > dev->max_queue_pairs) {
 602                 PMD_INIT_LOG(ERR, "multi-q config %u, but only %u supported",
 603                              q_pairs, dev->max_queue_pairs);
 604                 return -1;
 605         }
 606
 607         /* Server mode can't enable queue pairs if vhostfd is invalid,
 608          * always return 0 in this case.
 609          */
 610         if (!dev->is_server || dev->vhostfd >= 0) {
 611                 for (i = 0; i < q_pairs; ++i)
 612                         ret |= dev->ops->enable_qp(dev, i, 1);
 613                 for (i = q_pairs; i < dev->max_queue_pairs; ++i)
 614                         ret |= dev->ops->enable_qp(dev, i, 0);
 615         }
 616         dev->queue_pairs = q_pairs;
 617
 618         return ret;
 619 }
 620
 621 static uint32_t
 622 virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
 623                             uint16_t idx_hdr)
 624 {
 625         struct virtio_net_ctrl_hdr *hdr;
 626         virtio_net_ctrl_ack status = ~0;
 627         uint16_t i, idx_data, idx_status;
 628         uint32_t n_descs = 0;
 629
 630         /* locate desc for header, data, and status */
 631         idx_data = vring->desc[idx_hdr].next;
 632         n_descs++;
 633
 634         i = idx_data;
 635         while (vring->desc[i].flags == VRING_DESC_F_NEXT) {
 636                 i = vring->desc[i].next;
 637                 n_descs++;
 638         }
 639
 640         /* locate desc for status */
 641         idx_status = i;
 642         n_descs++;
 643
 644         hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr;
 645         if (hdr->class == VIRTIO_NET_CTRL_MQ &&
 646             hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
 647                 uint16_t queues;
 648
 649                 queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr;
 650                 status = virtio_user_handle_mq(dev, queues);
 651         } else if (hdr->class == VIRTIO_NET_CTRL_RX  ||
 652                    hdr->class == VIRTIO_NET_CTRL_MAC ||
 653                    hdr->class == VIRTIO_NET_CTRL_VLAN) {
 654                 status = 0;
 655         }
 656
 657         /* Update status */
 658         *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status;
 659
 660         return n_descs;
 661 }
 662
 663 static inline int
 664 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 665 {
 666         uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 667
 668         return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
 669                 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
 670 }
 671
 672 static uint32_t
 673 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev,
 674                                    struct vring_packed *vring,
 675                                    uint16_t idx_hdr)
 676 {
 677         struct virtio_net_ctrl_hdr *hdr;
 678         virtio_net_ctrl_ack status = ~0;
 679         uint16_t idx_data, idx_status;
 680         /* initialize to one, header is first */
 681         uint32_t n_descs = 1;
 682
 683         /* locate desc for header, data, and status */
 684         idx_data = idx_hdr + 1;
 685         if (idx_data >= dev->queue_size)
 686                 idx_data -= dev->queue_size;
 687
 688         n_descs++;
 689
 690         idx_status = idx_data;
 691         while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) {
 692                 idx_status++;
 693                 if (idx_status >= dev->queue_size)
 694                         idx_status -= dev->queue_size;
 695                 n_descs++;
 696         }
 697
 698         hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr;
 699         if (hdr->class == VIRTIO_NET_CTRL_MQ &&
 700             hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
 701                 uint16_t queues;
 702
 703                 queues = *(uint16_t *)(uintptr_t)
 704                                 vring->desc[idx_data].addr;
 705                 status = virtio_user_handle_mq(dev, queues);
 706         } else if (hdr->class == VIRTIO_NET_CTRL_RX  ||
 707                    hdr->class == VIRTIO_NET_CTRL_MAC ||
 708                    hdr->class == VIRTIO_NET_CTRL_VLAN) {
 709                 status = 0;
 710         }
 711
 712         /* Update status */
 713         *(virtio_net_ctrl_ack *)(uintptr_t)
 714                 vring->desc[idx_status].addr = status;
 715
 716         /* Update used descriptor */
 717         vring->desc[idx_hdr].id = vring->desc[idx_status].id;
 718         vring->desc[idx_hdr].len = sizeof(status);
 719
 720         return n_descs;
 721 }
 722
 723 void
 724 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 725 {
 726         struct virtio_user_queue *vq = &dev->packed_queues[queue_idx];
 727         struct vring_packed *vring = &dev->packed_vrings[queue_idx];
 728         uint16_t n_descs, flags;
 729
 730         /* Perform a load-acquire barrier in desc_is_avail to
 731          * enforce the ordering between desc flags and desc
 732          * content.
 733          */
 734         while (desc_is_avail(&vring->desc[vq->used_idx],
 735                              vq->used_wrap_counter)) {
 736
 737                 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring,
 738                                 vq->used_idx);
 739
 740                 flags = VRING_DESC_F_WRITE;
 741                 if (vq->used_wrap_counter)
 742                         flags |= VRING_PACKED_DESC_F_AVAIL_USED;
 743
 744                 __atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
 745                                  __ATOMIC_RELEASE);
 746
 747                 vq->used_idx += n_descs;
 748                 if (vq->used_idx >= dev->queue_size) {
 749                         vq->used_idx -= dev->queue_size;
 750                         vq->used_wrap_counter ^= 1;
 751                 }
 752         }
 753 }
 754
 755 void
 756 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx)
 757 {
 758         uint16_t avail_idx, desc_idx;
 759         struct vring_used_elem *uep;
 760         uint32_t n_descs;
 761         struct vring *vring = &dev->vrings[queue_idx];
 762
 763         /* Consume avail ring, using used ring idx as first one */
 764         while (__atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED)
 765                != vring->avail->idx) {
 766                 avail_idx = __atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED)
 767                             & (vring->num - 1);
 768                 desc_idx = vring->avail->ring[avail_idx];
 769
 770                 n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx);
 771
 772                 /* Update used ring */
 773                 uep = &vring->used->ring[avail_idx];
 774                 uep->id = desc_idx;
 775                 uep->len = n_descs;
 776
 777                 __atomic_add_fetch(&vring->used->idx, 1, __ATOMIC_RELAXED);
 778         }
 779 }
 780
 781 int
 782 virtio_user_send_status_update(struct virtio_user_dev *dev, uint8_t status)
 783 {
 784         int ret;
 785         uint64_t arg = status;
 786
 787         /* Vhost-user only for now */
 788         if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER)
 789                 return 0;
 790
 791         ret = dev->ops->send_request(dev, VHOST_USER_SET_STATUS, &arg);
 792         if (ret) {
 793                 PMD_INIT_LOG(ERR, "VHOST_USER_SET_STATUS failed (%d): %s", ret,
 794                              strerror(errno));
 795                 return -1;
 796         }
 797
 798         return 0;
 799 }
 800
 801 int
 802 virtio_user_update_status(struct virtio_user_dev *dev)
 803 {
 804         uint64_t ret;
 805         int err;
 806
 807         /* Vhost-user only for now */
 808         if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER)
 809                 return 0;
 810
 811         err = dev->ops->send_request(dev, VHOST_USER_GET_STATUS, &ret);
 812         if (err) {
 813                 PMD_INIT_LOG(ERR, "VHOST_USER_GET_STATUS failed (%d): %s", err,
 814                              strerror(errno));
 815                 return -1;
 816         }
 817         if (ret > UINT8_MAX) {
 818                 PMD_INIT_LOG(ERR, "Invalid VHOST_USER_GET_STATUS response 0x%" PRIx64 "\n", ret);
 819                 return -1;
 820         }
 821
 822         dev->status = ret;
 823         PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):\n"
 824                         "\t-RESET: %u\n"
 825                         "\t-ACKNOWLEDGE: %u\n"
 826                         "\t-DRIVER: %u\n"
 827                         "\t-DRIVER_OK: %u\n"
 828                         "\t-FEATURES_OK: %u\n"
 829                         "\t-DEVICE_NEED_RESET: %u\n"
 830                         "\t-FAILED: %u\n",
 831                         dev->status,
 832                         (dev->status == VIRTIO_CONFIG_STATUS_RESET),
 833                         !!(dev->status & VIRTIO_CONFIG_STATUS_ACK),
 834                         !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER),
 835                         !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK),
 836                         !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK),
 837                         !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET),
 838                         !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED));
 839         return 0;
 840 }