drivers/bus/auxiliary/auxiliary_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <string.h>
   6 #include <inttypes.h>
   7 #include <stdint.h>
   8 #include <stdbool.h>
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <sys/queue.h>
  12 #include <rte_errno.h>
  13 #include <rte_interrupts.h>
  14 #include <rte_log.h>
  15 #include <rte_bus.h>
  16 #include <rte_per_lcore.h>
  17 #include <rte_memory.h>
  18 #include <rte_eal.h>
  19 #include <rte_eal_paging.h>
  20 #include <rte_lcore.h>
  21 #include <rte_string_fns.h>
  22 #include <rte_common.h>
  23 #include <rte_devargs.h>
  24
  25 #include "private.h"
  26 #include "rte_bus_auxiliary.h"
  27
  28 static struct rte_devargs *
  29 auxiliary_devargs_lookup(const char *name)
  30 {
  31         struct rte_devargs *devargs;
  32
  33         RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
  34                 if (strcmp(devargs->name, name) == 0)
  35                         return devargs;
  36         }
  37         return NULL;
  38 }
  39
  40 /*
  41  * Test whether the auxiliary device exist.
  42  *
  43  * Stub for OS not supporting auxiliary bus.
  44  */
  45 __rte_weak bool
  46 auxiliary_dev_exists(const char *name)
  47 {
  48         RTE_SET_USED(name);
  49         return false;
  50 }
  51
  52 /*
  53  * Scan the devices in the auxiliary bus.
  54  *
  55  * Stub for OS not supporting auxiliary bus.
  56  */
  57 __rte_weak int
  58 auxiliary_scan(void)
  59 {
  60         return 0;
  61 }
  62
  63 /*
  64  * Update a device's devargs being scanned.
  65  */
  66 void
  67 auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
  68 {
  69         aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
  70 }
  71
  72 /*
  73  * Match the auxiliary driver and device using driver function.
  74  */
  75 bool
  76 auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
  77                 const struct rte_auxiliary_device *aux_dev)
  78 {
  79         if (aux_drv->match == NULL)
  80                 return false;
  81         return aux_drv->match(aux_dev->name);
  82 }
  83
  84 /*
  85  * Call the probe() function of the driver.
  86  */
  87 static int
  88 rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
  89                                struct rte_auxiliary_device *dev)
  90 {
  91         enum rte_iova_mode iova_mode;
  92         int ret;
  93
  94         if (drv == NULL || dev == NULL)
  95                 return -EINVAL;
  96
  97         /* Check if driver supports it. */
  98         if (!auxiliary_match(drv, dev))
  99                 /* Match of device and driver failed */
 100                 return 1;
 101
 102         /* No initialization when marked as blocked, return without error. */
 103         if (dev->device.devargs != NULL &&
 104             dev->device.devargs->policy == RTE_DEV_BLOCKED) {
 105                 AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
 106                 return -1;
 107         }
 108
 109         if (dev->device.numa_node < 0) {
 110                 if (rte_socket_count() > 1)
 111                         AUXILIARY_LOG(INFO, "Device %s is not NUMA-aware, defaulting socket to 0",
 112                                         dev->name);
 113                 dev->device.numa_node = 0;
 114         }
 115
 116         iova_mode = rte_eal_iova_mode();
 117         if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
 118             iova_mode != RTE_IOVA_VA) {
 119                 AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
 120                               drv->driver.name);
 121                 return -EINVAL;
 122         }
 123
 124         dev->driver = drv;
 125
 126         AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)",
 127                       drv->driver.name, dev->name, dev->device.numa_node);
 128         ret = drv->probe(drv, dev);
 129         if (ret != 0)
 130                 dev->driver = NULL;
 131         else
 132                 dev->device.driver = &drv->driver;
 133
 134         return ret;
 135 }
 136
 137 /*
 138  * Call the remove() function of the driver.
 139  */
 140 static int
 141 rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
 142 {
 143         struct rte_auxiliary_driver *drv;
 144         int ret = 0;
 145
 146         if (dev == NULL)
 147                 return -EINVAL;
 148
 149         drv = dev->driver;
 150
 151         AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i",
 152                       drv->driver.name, dev->name, dev->device.numa_node);
 153
 154         if (drv->remove != NULL) {
 155                 ret = drv->remove(dev);
 156                 if (ret < 0)
 157                         return ret;
 158         }
 159
 160         /* clear driver structure */
 161         dev->driver = NULL;
 162         dev->device.driver = NULL;
 163
 164         return 0;
 165 }
 166
 167 /*
 168  * Call the probe() function of all registered drivers for the given device.
 169  * Return < 0 if initialization failed.
 170  * Return 1 if no driver is found for this device.
 171  */
 172 static int
 173 auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
 174 {
 175         struct rte_auxiliary_driver *drv;
 176         int rc;
 177
 178         if (dev == NULL)
 179                 return -EINVAL;
 180
 181         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 182                 if (!drv->match(dev->name))
 183                         continue;
 184
 185                 rc = rte_auxiliary_probe_one_driver(drv, dev);
 186                 if (rc < 0)
 187                         /* negative value is an error */
 188                         return rc;
 189                 if (rc > 0)
 190                         /* positive value means driver doesn't support it */
 191                         continue;
 192                 return 0;
 193         }
 194         return 1;
 195 }
 196
 197 /*
 198  * Scan the content of the auxiliary bus, and call the probe function for
 199  * all registered drivers to try to probe discovered devices.
 200  */
 201 static int
 202 auxiliary_probe(void)
 203 {
 204         struct rte_auxiliary_device *dev = NULL;
 205         size_t probed = 0, failed = 0;
 206         int ret = 0;
 207
 208         FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
 209                 probed++;
 210
 211                 ret = auxiliary_probe_all_drivers(dev);
 212                 if (ret < 0) {
 213                         if (ret != -EEXIST) {
 214                                 AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
 215                                               dev->name);
 216                                 rte_errno = errno;
 217                                 failed++;
 218                         }
 219                         ret = 0;
 220                 }
 221         }
 222
 223         return (probed && probed == failed) ? -1 : 0;
 224 }
 225
 226 static int
 227 auxiliary_parse(const char *name, void *addr)
 228 {
 229         struct rte_auxiliary_driver *drv = NULL;
 230         const char **out = addr;
 231
 232         /* Allow empty device name "auxiliary:" to bypass entire bus scan. */
 233         if (strlen(name) == 0)
 234                 return 0;
 235
 236         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 237                 if (drv->match(name))
 238                         break;
 239         }
 240         if (drv != NULL && addr != NULL)
 241                 *out = name;
 242         return drv != NULL ? 0 : -1;
 243 }
 244
 245 /* Register a driver */
 246 void
 247 rte_auxiliary_register(struct rte_auxiliary_driver *driver)
 248 {
 249         TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
 250         driver->bus = &auxiliary_bus;
 251 }
 252
 253 /* Unregister a driver */
 254 void
 255 rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
 256 {
 257         TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
 258         driver->bus = NULL;
 259 }
 260
 261 /* Add a device to auxiliary bus */
 262 void
 263 auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
 264 {
 265         TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
 266 }
 267
 268 /* Insert a device into a predefined position in auxiliary bus */
 269 void
 270 auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
 271                         struct rte_auxiliary_device *new_aux_dev)
 272 {
 273         TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
 274 }
 275
 276 /* Remove a device from auxiliary bus */
 277 static void
 278 rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
 279 {
 280         TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
 281 }
 282
 283 static struct rte_device *
 284 auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 285                       const void *data)
 286 {
 287         const struct rte_auxiliary_device *pstart;
 288         struct rte_auxiliary_device *adev;
 289
 290         if (start != NULL) {
 291                 pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
 292                 adev = TAILQ_NEXT(pstart, next);
 293         } else {
 294                 adev = TAILQ_FIRST(&auxiliary_bus.device_list);
 295         }
 296         while (adev != NULL) {
 297                 if (cmp(&adev->device, data) == 0)
 298                         return &adev->device;
 299                 adev = TAILQ_NEXT(adev, next);
 300         }
 301         return NULL;
 302 }
 303
 304 static int
 305 auxiliary_plug(struct rte_device *dev)
 306 {
 307         if (!auxiliary_dev_exists(dev->name))
 308                 return -ENOENT;
 309         return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
 310 }
 311
 312 static int
 313 auxiliary_unplug(struct rte_device *dev)
 314 {
 315         struct rte_auxiliary_device *adev;
 316         int ret;
 317
 318         adev = RTE_DEV_TO_AUXILIARY(dev);
 319         ret = rte_auxiliary_driver_remove_dev(adev);
 320         if (ret == 0) {
 321                 rte_auxiliary_remove_device(adev);
 322                 rte_devargs_remove(dev->devargs);
 323                 free(adev);
 324         }
 325         return ret;
 326 }
 327
 328 static int
 329 auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
 330 {
 331         struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
 332
 333         if (dev == NULL || aux_dev->driver == NULL) {
 334                 rte_errno = EINVAL;
 335                 return -1;
 336         }
 337         if (aux_dev->driver->dma_map == NULL) {
 338                 rte_errno = ENOTSUP;
 339                 return -1;
 340         }
 341         return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
 342 }
 343
 344 static int
 345 auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
 346                     size_t len)
 347 {
 348         struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
 349
 350         if (dev == NULL || aux_dev->driver == NULL) {
 351                 rte_errno = EINVAL;
 352                 return -1;
 353         }
 354         if (aux_dev->driver->dma_unmap == NULL) {
 355                 rte_errno = ENOTSUP;
 356                 return -1;
 357         }
 358         return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
 359 }
 360
 361 bool
 362 auxiliary_is_ignored_device(const char *name)
 363 {
 364         struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
 365
 366         switch (auxiliary_bus.bus.conf.scan_mode) {
 367         case RTE_BUS_SCAN_ALLOWLIST:
 368                 if (devargs && devargs->policy == RTE_DEV_ALLOWED)
 369                         return false;
 370                 break;
 371         case RTE_BUS_SCAN_UNDEFINED:
 372         case RTE_BUS_SCAN_BLOCKLIST:
 373                 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
 374                         return false;
 375                 break;
 376         }
 377         return true;
 378 }
 379
 380 static enum rte_iova_mode
 381 auxiliary_get_iommu_class(void)
 382 {
 383         const struct rte_auxiliary_driver *drv;
 384
 385         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 386                 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
 387                         return RTE_IOVA_VA;
 388         }
 389
 390         return RTE_IOVA_DC;
 391 }
 392
 393 struct rte_auxiliary_bus auxiliary_bus = {
 394         .bus = {
 395                 .scan = auxiliary_scan,
 396                 .probe = auxiliary_probe,
 397                 .find_device = auxiliary_find_device,
 398                 .plug = auxiliary_plug,
 399                 .unplug = auxiliary_unplug,
 400                 .parse = auxiliary_parse,
 401                 .dma_map = auxiliary_dma_map,
 402                 .dma_unmap = auxiliary_dma_unmap,
 403                 .get_iommu_class = auxiliary_get_iommu_class,
 404                 .dev_iterate = auxiliary_dev_iterate,
 405         },
 406         .device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
 407         .driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
 408 };
 409
 410 RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
 411 RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);