drivers/bus/vdev/vdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016 RehiveTech. All rights reserved.
   3  */
   4
   5 #include <string.h>
   6 #include <inttypes.h>
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <stdint.h>
  10 #include <stdbool.h>
  11 #include <sys/queue.h>
  12
  13 #include <rte_eal.h>
  14 #include <rte_dev.h>
  15 #include <rte_bus.h>
  16 #include <rte_common.h>
  17 #include <rte_devargs.h>
  18 #include <rte_memory.h>
  19 #include <rte_tailq.h>
  20 #include <rte_spinlock.h>
  21 #include <rte_string_fns.h>
  22 #include <rte_errno.h>
  23
  24 #include "rte_bus_vdev.h"
  25 #include "vdev_logs.h"
  26 #include "vdev_private.h"
  27
  28 #define VDEV_MP_KEY     "bus_vdev_mp"
  29
  30 /* Forward declare to access virtual bus name */
  31 static struct rte_bus rte_vdev_bus;
  32
  33 /** Double linked list of virtual device drivers. */
  34 TAILQ_HEAD(vdev_device_list, rte_vdev_device);
  35
  36 static struct vdev_device_list vdev_device_list =
  37         TAILQ_HEAD_INITIALIZER(vdev_device_list);
  38 /* The lock needs to be recursive because a vdev can manage another vdev. */
  39 static rte_spinlock_recursive_t vdev_device_list_lock =
  40         RTE_SPINLOCK_RECURSIVE_INITIALIZER;
  41
  42 static struct vdev_driver_list vdev_driver_list =
  43         TAILQ_HEAD_INITIALIZER(vdev_driver_list);
  44
  45 struct vdev_custom_scan {
  46         TAILQ_ENTRY(vdev_custom_scan) next;
  47         rte_vdev_scan_callback callback;
  48         void *user_arg;
  49 };
  50 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan);
  51 static struct vdev_custom_scans vdev_custom_scans =
  52         TAILQ_HEAD_INITIALIZER(vdev_custom_scans);
  53 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER;
  54
  55 /* register a driver */
  56 void
  57 rte_vdev_register(struct rte_vdev_driver *driver)
  58 {
  59         TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next);
  60 }
  61
  62 /* unregister a driver */
  63 void
  64 rte_vdev_unregister(struct rte_vdev_driver *driver)
  65 {
  66         TAILQ_REMOVE(&vdev_driver_list, driver, next);
  67 }
  68
  69 int
  70 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
  71 {
  72         struct vdev_custom_scan *custom_scan;
  73
  74         rte_spinlock_lock(&vdev_custom_scan_lock);
  75
  76         /* check if already registered */
  77         TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
  78                 if (custom_scan->callback == callback &&
  79                                 custom_scan->user_arg == user_arg)
  80                         break;
  81         }
  82
  83         if (custom_scan == NULL) {
  84                 custom_scan = malloc(sizeof(struct vdev_custom_scan));
  85                 if (custom_scan != NULL) {
  86                         custom_scan->callback = callback;
  87                         custom_scan->user_arg = user_arg;
  88                         TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next);
  89                 }
  90         }
  91
  92         rte_spinlock_unlock(&vdev_custom_scan_lock);
  93
  94         return (custom_scan == NULL) ? -1 : 0;
  95 }
  96
  97 int
  98 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
  99 {
 100         struct vdev_custom_scan *custom_scan, *tmp_scan;
 101
 102         rte_spinlock_lock(&vdev_custom_scan_lock);
 103         RTE_TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next,
 104                                 tmp_scan) {
 105                 if (custom_scan->callback != callback ||
 106                                 (custom_scan->user_arg != (void *)-1 &&
 107                                 custom_scan->user_arg != user_arg))
 108                         continue;
 109                 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next);
 110                 free(custom_scan);
 111         }
 112         rte_spinlock_unlock(&vdev_custom_scan_lock);
 113
 114         return 0;
 115 }
 116
 117 static int
 118 vdev_parse(const char *name, void *addr)
 119 {
 120         struct rte_vdev_driver **out = addr;
 121         struct rte_vdev_driver *driver = NULL;
 122
 123         TAILQ_FOREACH(driver, &vdev_driver_list, next) {
 124                 if (strncmp(driver->driver.name, name,
 125                             strlen(driver->driver.name)) == 0)
 126                         break;
 127                 if (driver->driver.alias &&
 128                     strncmp(driver->driver.alias, name,
 129                             strlen(driver->driver.alias)) == 0)
 130                         break;
 131         }
 132         if (driver != NULL &&
 133             addr != NULL)
 134                 *out = driver;
 135         return driver == NULL;
 136 }
 137
 138 static int
 139 vdev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
 140 {
 141         struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev);
 142         const struct rte_vdev_driver *driver;
 143
 144         if (!vdev) {
 145                 rte_errno = EINVAL;
 146                 return -1;
 147         }
 148
 149         if (!vdev->device.driver) {
 150                 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name);
 151                 return 1;
 152         }
 153
 154         driver = container_of(vdev->device.driver, const struct rte_vdev_driver,
 155                         driver);
 156
 157         if (driver->dma_map)
 158                 return driver->dma_map(vdev, addr, iova, len);
 159
 160         return 0;
 161 }
 162
 163 static int
 164 vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
 165 {
 166         struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev);
 167         const struct rte_vdev_driver *driver;
 168
 169         if (!vdev) {
 170                 rte_errno = EINVAL;
 171                 return -1;
 172         }
 173
 174         if (!vdev->device.driver) {
 175                 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name);
 176                 return 1;
 177         }
 178
 179         driver = container_of(vdev->device.driver, const struct rte_vdev_driver,
 180                         driver);
 181
 182         if (driver->dma_unmap)
 183                 return driver->dma_unmap(vdev, addr, iova, len);
 184
 185         return 0;
 186 }
 187
 188 static int
 189 vdev_probe_all_drivers(struct rte_vdev_device *dev)
 190 {
 191         const char *name;
 192         struct rte_vdev_driver *driver;
 193         enum rte_iova_mode iova_mode;
 194         int ret;
 195
 196         if (rte_dev_is_probed(&dev->device))
 197                 return -EEXIST;
 198
 199         name = rte_vdev_device_name(dev);
 200         VDEV_LOG(DEBUG, "Search driver to probe device %s", name);
 201
 202         if (vdev_parse(name, &driver))
 203                 return -1;
 204
 205         iova_mode = rte_eal_iova_mode();
 206         if ((driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) && (iova_mode == RTE_IOVA_PA)) {
 207                 VDEV_LOG(ERR, "%s requires VA IOVA mode but current mode is PA, not initializing",
 208                                 name);
 209                 return -1;
 210         }
 211
 212         ret = driver->probe(dev);
 213         if (ret == 0)
 214                 dev->device.driver = &driver->driver;
 215         return ret;
 216 }
 217
 218 /* The caller shall be responsible for thread-safe */
 219 static struct rte_vdev_device *
 220 find_vdev(const char *name)
 221 {
 222         struct rte_vdev_device *dev;
 223
 224         if (!name)
 225                 return NULL;
 226
 227         TAILQ_FOREACH(dev, &vdev_device_list, next) {
 228                 const char *devname = rte_vdev_device_name(dev);
 229
 230                 if (!strcmp(devname, name))
 231                         return dev;
 232         }
 233
 234         return NULL;
 235 }
 236
 237 static struct rte_devargs *
 238 alloc_devargs(const char *name, const char *args)
 239 {
 240         struct rte_devargs *devargs;
 241         int ret;
 242
 243         devargs = calloc(1, sizeof(*devargs));
 244         if (!devargs)
 245                 return NULL;
 246
 247         devargs->bus = &rte_vdev_bus;
 248         if (args)
 249                 devargs->data = strdup(args);
 250         else
 251                 devargs->data = strdup("");
 252         devargs->args = devargs->data;
 253
 254         ret = strlcpy(devargs->name, name, sizeof(devargs->name));
 255         if (ret < 0 || ret >= (int)sizeof(devargs->name)) {
 256                 rte_devargs_reset(devargs);
 257                 free(devargs);
 258                 return NULL;
 259         }
 260
 261         return devargs;
 262 }
 263
 264 static int
 265 insert_vdev(const char *name, const char *args,
 266                 struct rte_vdev_device **p_dev,
 267                 bool init)
 268 {
 269         struct rte_vdev_device *dev;
 270         struct rte_devargs *devargs;
 271         int ret;
 272
 273         if (name == NULL)
 274                 return -EINVAL;
 275
 276         devargs = alloc_devargs(name, args);
 277         if (!devargs)
 278                 return -ENOMEM;
 279
 280         dev = calloc(1, sizeof(*dev));
 281         if (!dev) {
 282                 ret = -ENOMEM;
 283                 goto fail;
 284         }
 285
 286         dev->device.bus = &rte_vdev_bus;
 287         dev->device.numa_node = SOCKET_ID_ANY;
 288         dev->device.name = devargs->name;
 289
 290         if (find_vdev(name)) {
 291                 /*
 292                  * A vdev is expected to have only one port.
 293                  * So there is no reason to try probing again,
 294                  * even with new arguments.
 295                  */
 296                 ret = -EEXIST;
 297                 goto fail;
 298         }
 299
 300         if (init)
 301                 rte_devargs_insert(&devargs);
 302         dev->device.devargs = devargs;
 303         TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
 304
 305         if (p_dev)
 306                 *p_dev = dev;
 307
 308         return 0;
 309 fail:
 310         rte_devargs_reset(devargs);
 311         free(devargs);
 312         free(dev);
 313         return ret;
 314 }
 315
 316 int
 317 rte_vdev_init(const char *name, const char *args)
 318 {
 319         struct rte_vdev_device *dev;
 320         int ret;
 321
 322         rte_spinlock_recursive_lock(&vdev_device_list_lock);
 323         ret = insert_vdev(name, args, &dev, true);
 324         if (ret == 0) {
 325                 ret = vdev_probe_all_drivers(dev);
 326                 if (ret) {
 327                         if (ret > 0)
 328                                 VDEV_LOG(ERR, "no driver found for %s", name);
 329                         /* If fails, remove it from vdev list */
 330                         TAILQ_REMOVE(&vdev_device_list, dev, next);
 331                         rte_devargs_remove(dev->device.devargs);
 332                         free(dev);
 333                 }
 334         }
 335         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 336         return ret;
 337 }
 338
 339 static int
 340 vdev_remove_driver(struct rte_vdev_device *dev)
 341 {
 342         const char *name = rte_vdev_device_name(dev);
 343         const struct rte_vdev_driver *driver;
 344
 345         if (!dev->device.driver) {
 346                 VDEV_LOG(DEBUG, "no driver attach to device %s", name);
 347                 return 1;
 348         }
 349
 350         driver = container_of(dev->device.driver, const struct rte_vdev_driver,
 351                 driver);
 352         return driver->remove(dev);
 353 }
 354
 355 int
 356 rte_vdev_uninit(const char *name)
 357 {
 358         struct rte_vdev_device *dev;
 359         int ret;
 360
 361         if (name == NULL)
 362                 return -EINVAL;
 363
 364         rte_spinlock_recursive_lock(&vdev_device_list_lock);
 365
 366         dev = find_vdev(name);
 367         if (!dev) {
 368                 ret = -ENOENT;
 369                 goto unlock;
 370         }
 371
 372         ret = vdev_remove_driver(dev);
 373         if (ret)
 374                 goto unlock;
 375
 376         TAILQ_REMOVE(&vdev_device_list, dev, next);
 377         rte_devargs_remove(dev->device.devargs);
 378         free(dev);
 379
 380 unlock:
 381         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 382         return ret;
 383 }
 384
 385 struct vdev_param {
 386 #define VDEV_SCAN_REQ   1
 387 #define VDEV_SCAN_ONE   2
 388 #define VDEV_SCAN_REP   3
 389         int type;
 390         int num;
 391         char name[RTE_DEV_NAME_MAX_LEN];
 392 };
 393
 394 static int vdev_plug(struct rte_device *dev);
 395
 396 /**
 397  * This function works as the action for both primary and secondary process
 398  * for static vdev discovery when a secondary process is booting.
 399  *
 400  * step 1, secondary process sends a sync request to ask for vdev in primary;
 401  * step 2, primary process receives the request, and send vdevs one by one;
 402  * step 3, primary process sends back reply, which indicates how many vdevs
 403  * are sent.
 404  */
 405 static int
 406 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer)
 407 {
 408         struct rte_vdev_device *dev;
 409         struct rte_mp_msg mp_resp;
 410         struct vdev_param *ou = (struct vdev_param *)&mp_resp.param;
 411         const struct vdev_param *in = (const struct vdev_param *)mp_msg->param;
 412         const char *devname;
 413         int num;
 414         int ret;
 415
 416         strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name));
 417         mp_resp.len_param = sizeof(*ou);
 418         mp_resp.num_fds = 0;
 419
 420         switch (in->type) {
 421         case VDEV_SCAN_REQ:
 422                 ou->type = VDEV_SCAN_ONE;
 423                 ou->num = 1;
 424                 num = 0;
 425
 426                 rte_spinlock_recursive_lock(&vdev_device_list_lock);
 427                 TAILQ_FOREACH(dev, &vdev_device_list, next) {
 428                         devname = rte_vdev_device_name(dev);
 429                         if (strlen(devname) == 0) {
 430                                 VDEV_LOG(INFO, "vdev with no name is not sent");
 431                                 continue;
 432                         }
 433                         VDEV_LOG(INFO, "send vdev, %s", devname);
 434                         strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN);
 435                         if (rte_mp_sendmsg(&mp_resp) < 0)
 436                                 VDEV_LOG(ERR, "send vdev, %s, failed, %s",
 437                                          devname, strerror(rte_errno));
 438                         num++;
 439                 }
 440                 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 441
 442                 ou->type = VDEV_SCAN_REP;
 443                 ou->num = num;
 444                 if (rte_mp_reply(&mp_resp, peer) < 0)
 445                         VDEV_LOG(ERR, "Failed to reply a scan request");
 446                 break;
 447         case VDEV_SCAN_ONE:
 448                 VDEV_LOG(INFO, "receive vdev, %s", in->name);
 449                 ret = insert_vdev(in->name, NULL, NULL, false);
 450                 if (ret == -EEXIST)
 451                         VDEV_LOG(DEBUG, "device already exist, %s", in->name);
 452                 else if (ret < 0)
 453                         VDEV_LOG(ERR, "failed to add vdev, %s", in->name);
 454                 break;
 455         default:
 456                 VDEV_LOG(ERR, "vdev cannot recognize this message");
 457         }
 458
 459         return 0;
 460 }
 461
 462 static int
 463 vdev_scan(void)
 464 {
 465         struct rte_vdev_device *dev;
 466         struct rte_devargs *devargs;
 467         struct vdev_custom_scan *custom_scan;
 468
 469         if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 &&
 470             rte_errno != EEXIST) {
 471                 /* for primary, unsupported IPC is not an error */
 472                 if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
 473                                 rte_errno == ENOTSUP)
 474                         goto scan;
 475                 VDEV_LOG(ERR, "Failed to add vdev mp action");
 476                 return -1;
 477         }
 478
 479         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 480                 struct rte_mp_msg mp_req, *mp_rep;
 481                 struct rte_mp_reply mp_reply;
 482                 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
 483                 struct vdev_param *req = (struct vdev_param *)mp_req.param;
 484                 struct vdev_param *resp;
 485
 486                 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name));
 487                 mp_req.len_param = sizeof(*req);
 488                 mp_req.num_fds = 0;
 489                 req->type = VDEV_SCAN_REQ;
 490                 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
 491                     mp_reply.nb_received == 1) {
 492                         mp_rep = &mp_reply.msgs[0];
 493                         resp = (struct vdev_param *)mp_rep->param;
 494                         VDEV_LOG(INFO, "Received %d vdevs", resp->num);
 495                         free(mp_reply.msgs);
 496                 } else
 497                         VDEV_LOG(ERR, "Failed to request vdev from primary");
 498
 499                 /* Fall through to allow private vdevs in secondary process */
 500         }
 501
 502 scan:
 503         /* call custom scan callbacks if any */
 504         rte_spinlock_lock(&vdev_custom_scan_lock);
 505         TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
 506                 if (custom_scan->callback != NULL)
 507                         /*
 508                          * the callback should update devargs list
 509                          * by calling rte_devargs_insert() with
 510                          *     devargs.bus = rte_bus_find_by_name("vdev");
 511                          *     devargs.type = RTE_DEVTYPE_VIRTUAL;
 512                          *     devargs.policy = RTE_DEV_ALLOWED;
 513                          */
 514                         custom_scan->callback(custom_scan->user_arg);
 515         }
 516         rte_spinlock_unlock(&vdev_custom_scan_lock);
 517
 518         /* for virtual devices we scan the devargs_list populated via cmdline */
 519         RTE_EAL_DEVARGS_FOREACH("vdev", devargs) {
 520
 521                 dev = calloc(1, sizeof(*dev));
 522                 if (!dev)
 523                         return -1;
 524
 525                 rte_spinlock_recursive_lock(&vdev_device_list_lock);
 526
 527                 if (find_vdev(devargs->name)) {
 528                         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 529                         free(dev);
 530                         continue;
 531                 }
 532
 533                 dev->device.bus = &rte_vdev_bus;
 534                 dev->device.devargs = devargs;
 535                 dev->device.numa_node = SOCKET_ID_ANY;
 536                 dev->device.name = devargs->name;
 537
 538                 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
 539
 540                 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 541         }
 542
 543         return 0;
 544 }
 545
 546 static int
 547 vdev_probe(void)
 548 {
 549         struct rte_vdev_device *dev;
 550         int r, ret = 0;
 551
 552         /* call the init function for each virtual device */
 553         TAILQ_FOREACH(dev, &vdev_device_list, next) {
 554                 /* we don't use the vdev lock here, as it's only used in DPDK
 555                  * initialization; and we don't want to hold such a lock when
 556                  * we call each driver probe.
 557                  */
 558
 559                 r = vdev_probe_all_drivers(dev);
 560                 if (r != 0) {
 561                         if (r == -EEXIST)
 562                                 continue;
 563                         VDEV_LOG(ERR, "failed to initialize %s device",
 564                                 rte_vdev_device_name(dev));
 565                         ret = -1;
 566                 }
 567         }
 568
 569         return ret;
 570 }
 571
 572 struct rte_device *
 573 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 574                      const void *data)
 575 {
 576         const struct rte_vdev_device *vstart;
 577         struct rte_vdev_device *dev;
 578
 579         rte_spinlock_recursive_lock(&vdev_device_list_lock);
 580         if (start != NULL) {
 581                 vstart = RTE_DEV_TO_VDEV_CONST(start);
 582                 dev = TAILQ_NEXT(vstart, next);
 583         } else {
 584                 dev = TAILQ_FIRST(&vdev_device_list);
 585         }
 586         while (dev != NULL) {
 587                 if (cmp(&dev->device, data) == 0)
 588                         break;
 589                 dev = TAILQ_NEXT(dev, next);
 590         }
 591         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 592
 593         return dev ? &dev->device : NULL;
 594 }
 595
 596 static int
 597 vdev_plug(struct rte_device *dev)
 598 {
 599         return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev));
 600 }
 601
 602 static int
 603 vdev_unplug(struct rte_device *dev)
 604 {
 605         return rte_vdev_uninit(dev->name);
 606 }
 607
 608 static enum rte_iova_mode
 609 vdev_get_iommu_class(void)
 610 {
 611         const char *name;
 612         struct rte_vdev_device *dev;
 613         struct rte_vdev_driver *driver;
 614
 615         TAILQ_FOREACH(dev, &vdev_device_list, next) {
 616                 name = rte_vdev_device_name(dev);
 617                 if (vdev_parse(name, &driver))
 618                         continue;
 619
 620                 if (driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA)
 621                         return RTE_IOVA_VA;
 622         }
 623
 624         return RTE_IOVA_DC;
 625 }
 626
 627 static struct rte_bus rte_vdev_bus = {
 628         .scan = vdev_scan,
 629         .probe = vdev_probe,
 630         .find_device = rte_vdev_find_device,
 631         .plug = vdev_plug,
 632         .unplug = vdev_unplug,
 633         .parse = vdev_parse,
 634         .dma_map = vdev_dma_map,
 635         .dma_unmap = vdev_dma_unmap,
 636         .get_iommu_class = vdev_get_iommu_class,
 637         .dev_iterate = rte_vdev_dev_iterate,
 638 };
 639
 640 RTE_REGISTER_BUS(vdev, rte_vdev_bus);
 641 RTE_LOG_REGISTER_DEFAULT(vdev_logtype_bus, NOTICE);