drivers/bus/auxiliary/auxiliary_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <string.h>
   6 #include <inttypes.h>
   7 #include <stdint.h>
   8 #include <stdbool.h>
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <sys/queue.h>
  12 #include <rte_errno.h>
  13 #include <rte_interrupts.h>
  14 #include <rte_log.h>
  15 #include <rte_bus.h>
  16 #include <rte_per_lcore.h>
  17 #include <rte_memory.h>
  18 #include <rte_eal.h>
  19 #include <rte_eal_paging.h>
  20 #include <rte_lcore.h>
  21 #include <rte_string_fns.h>
  22 #include <rte_common.h>
  23 #include <rte_devargs.h>
  24
  25 #include "private.h"
  26 #include "rte_bus_auxiliary.h"
  27
  28 static struct rte_devargs *
  29 auxiliary_devargs_lookup(const char *name)
  30 {
  31         struct rte_devargs *devargs;
  32
  33         RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
  34                 if (strcmp(devargs->name, name) == 0)
  35                         return devargs;
  36         }
  37         return NULL;
  38 }
  39
  40 /*
  41  * Test whether the auxiliary device exist.
  42  *
  43  * Stub for OS not supporting auxiliary bus.
  44  */
  45 __rte_weak bool
  46 auxiliary_dev_exists(const char *name)
  47 {
  48         RTE_SET_USED(name);
  49         return false;
  50 }
  51
  52 /*
  53  * Scan the devices in the auxiliary bus.
  54  *
  55  * Stub for OS not supporting auxiliary bus.
  56  */
  57 __rte_weak int
  58 auxiliary_scan(void)
  59 {
  60         return 0;
  61 }
  62
  63 /*
  64  * Update a device's devargs being scanned.
  65  */
  66 void
  67 auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
  68 {
  69         aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
  70 }
  71
  72 /*
  73  * Match the auxiliary driver and device using driver function.
  74  */
  75 bool
  76 auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
  77                 const struct rte_auxiliary_device *aux_dev)
  78 {
  79         if (aux_drv->match == NULL)
  80                 return false;
  81         return aux_drv->match(aux_dev->name);
  82 }
  83
  84 /*
  85  * Call the probe() function of the driver.
  86  */
  87 static int
  88 rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
  89                                struct rte_auxiliary_device *dev)
  90 {
  91         enum rte_iova_mode iova_mode;
  92         int ret;
  93
  94         if (drv == NULL || dev == NULL)
  95                 return -EINVAL;
  96
  97         /* Check if driver supports it. */
  98         if (!auxiliary_match(drv, dev))
  99                 /* Match of device and driver failed */
 100                 return 1;
 101
 102         /* No initialization when marked as blocked, return without error. */
 103         if (dev->device.devargs != NULL &&
 104             dev->device.devargs->policy == RTE_DEV_BLOCKED) {
 105                 AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
 106                 return -1;
 107         }
 108
 109         if (dev->device.numa_node < 0) {
 110                 if (rte_socket_count() > 1)
 111                         AUXILIARY_LOG(INFO, "Device %s is not NUMA-aware, defaulting socket to 0",
 112                                         dev->name);
 113                 dev->device.numa_node = 0;
 114         }
 115
 116         iova_mode = rte_eal_iova_mode();
 117         if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
 118             iova_mode != RTE_IOVA_VA) {
 119                 AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
 120                               drv->driver.name);
 121                 return -EINVAL;
 122         }
 123
 124         /* Allocate interrupt instance */
 125         dev->intr_handle =
 126                 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE);
 127         if (dev->intr_handle == NULL) {
 128                 AUXILIARY_LOG(ERR, "Could not allocate interrupt instance for device %s",
 129                         dev->name);
 130                 return -ENOMEM;
 131         }
 132
 133         dev->driver = drv;
 134
 135         AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)",
 136                       drv->driver.name, dev->name, dev->device.numa_node);
 137         ret = drv->probe(drv, dev);
 138         if (ret != 0) {
 139                 dev->driver = NULL;
 140                 rte_intr_instance_free(dev->intr_handle);
 141                 dev->intr_handle = NULL;
 142         } else {
 143                 dev->device.driver = &drv->driver;
 144         }
 145
 146         return ret;
 147 }
 148
 149 /*
 150  * Call the remove() function of the driver.
 151  */
 152 static int
 153 rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
 154 {
 155         struct rte_auxiliary_driver *drv;
 156         int ret = 0;
 157
 158         if (dev == NULL)
 159                 return -EINVAL;
 160
 161         drv = dev->driver;
 162
 163         AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i",
 164                       drv->driver.name, dev->name, dev->device.numa_node);
 165
 166         if (drv->remove != NULL) {
 167                 ret = drv->remove(dev);
 168                 if (ret < 0)
 169                         return ret;
 170         }
 171
 172         /* clear driver structure */
 173         dev->driver = NULL;
 174         dev->device.driver = NULL;
 175
 176         return 0;
 177 }
 178
 179 /*
 180  * Call the probe() function of all registered drivers for the given device.
 181  * Return < 0 if initialization failed.
 182  * Return 1 if no driver is found for this device.
 183  */
 184 static int
 185 auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
 186 {
 187         struct rte_auxiliary_driver *drv;
 188         int rc;
 189
 190         if (dev == NULL)
 191                 return -EINVAL;
 192
 193         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 194                 if (!drv->match(dev->name))
 195                         continue;
 196
 197                 rc = rte_auxiliary_probe_one_driver(drv, dev);
 198                 if (rc < 0)
 199                         /* negative value is an error */
 200                         return rc;
 201                 if (rc > 0)
 202                         /* positive value means driver doesn't support it */
 203                         continue;
 204                 return 0;
 205         }
 206         return 1;
 207 }
 208
 209 /*
 210  * Scan the content of the auxiliary bus, and call the probe function for
 211  * all registered drivers to try to probe discovered devices.
 212  */
 213 static int
 214 auxiliary_probe(void)
 215 {
 216         struct rte_auxiliary_device *dev = NULL;
 217         size_t probed = 0, failed = 0;
 218         int ret = 0;
 219
 220         FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
 221                 probed++;
 222
 223                 ret = auxiliary_probe_all_drivers(dev);
 224                 if (ret < 0) {
 225                         if (ret != -EEXIST) {
 226                                 AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
 227                                               dev->name);
 228                                 rte_errno = errno;
 229                                 failed++;
 230                         }
 231                         ret = 0;
 232                 }
 233         }
 234
 235         return (probed && probed == failed) ? -1 : 0;
 236 }
 237
 238 static int
 239 auxiliary_parse(const char *name, void *addr)
 240 {
 241         struct rte_auxiliary_driver *drv = NULL;
 242         const char **out = addr;
 243
 244         /* Allow empty device name "auxiliary:" to bypass entire bus scan. */
 245         if (strlen(name) == 0)
 246                 return 0;
 247
 248         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 249                 if (drv->match(name))
 250                         break;
 251         }
 252         if (drv != NULL && addr != NULL)
 253                 *out = name;
 254         return drv != NULL ? 0 : -1;
 255 }
 256
 257 /* Register a driver */
 258 void
 259 rte_auxiliary_register(struct rte_auxiliary_driver *driver)
 260 {
 261         TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
 262         driver->bus = &auxiliary_bus;
 263 }
 264
 265 /* Unregister a driver */
 266 void
 267 rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
 268 {
 269         TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
 270         driver->bus = NULL;
 271 }
 272
 273 /* Add a device to auxiliary bus */
 274 void
 275 auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
 276 {
 277         TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
 278 }
 279
 280 /* Insert a device into a predefined position in auxiliary bus */
 281 void
 282 auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
 283                         struct rte_auxiliary_device *new_aux_dev)
 284 {
 285         TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
 286 }
 287
 288 /* Remove a device from auxiliary bus */
 289 static void
 290 rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
 291 {
 292         TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
 293 }
 294
 295 static struct rte_device *
 296 auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 297                       const void *data)
 298 {
 299         const struct rte_auxiliary_device *pstart;
 300         struct rte_auxiliary_device *adev;
 301
 302         if (start != NULL) {
 303                 pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
 304                 adev = TAILQ_NEXT(pstart, next);
 305         } else {
 306                 adev = TAILQ_FIRST(&auxiliary_bus.device_list);
 307         }
 308         while (adev != NULL) {
 309                 if (cmp(&adev->device, data) == 0)
 310                         return &adev->device;
 311                 adev = TAILQ_NEXT(adev, next);
 312         }
 313         return NULL;
 314 }
 315
 316 static int
 317 auxiliary_plug(struct rte_device *dev)
 318 {
 319         if (!auxiliary_dev_exists(dev->name))
 320                 return -ENOENT;
 321         return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
 322 }
 323
 324 static int
 325 auxiliary_unplug(struct rte_device *dev)
 326 {
 327         struct rte_auxiliary_device *adev;
 328         int ret;
 329
 330         adev = RTE_DEV_TO_AUXILIARY(dev);
 331         ret = rte_auxiliary_driver_remove_dev(adev);
 332         if (ret == 0) {
 333                 rte_auxiliary_remove_device(adev);
 334                 rte_devargs_remove(dev->devargs);
 335                 rte_intr_instance_free(adev->intr_handle);
 336                 free(adev);
 337         }
 338         return ret;
 339 }
 340
 341 static int
 342 auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
 343 {
 344         struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
 345
 346         if (dev == NULL || aux_dev->driver == NULL) {
 347                 rte_errno = EINVAL;
 348                 return -1;
 349         }
 350         if (aux_dev->driver->dma_map == NULL) {
 351                 rte_errno = ENOTSUP;
 352                 return -1;
 353         }
 354         return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
 355 }
 356
 357 static int
 358 auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
 359                     size_t len)
 360 {
 361         struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
 362
 363         if (dev == NULL || aux_dev->driver == NULL) {
 364                 rte_errno = EINVAL;
 365                 return -1;
 366         }
 367         if (aux_dev->driver->dma_unmap == NULL) {
 368                 rte_errno = ENOTSUP;
 369                 return -1;
 370         }
 371         return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
 372 }
 373
 374 bool
 375 auxiliary_is_ignored_device(const char *name)
 376 {
 377         struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
 378
 379         switch (auxiliary_bus.bus.conf.scan_mode) {
 380         case RTE_BUS_SCAN_ALLOWLIST:
 381                 if (devargs && devargs->policy == RTE_DEV_ALLOWED)
 382                         return false;
 383                 break;
 384         case RTE_BUS_SCAN_UNDEFINED:
 385         case RTE_BUS_SCAN_BLOCKLIST:
 386                 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
 387                         return false;
 388                 break;
 389         }
 390         return true;
 391 }
 392
 393 static enum rte_iova_mode
 394 auxiliary_get_iommu_class(void)
 395 {
 396         const struct rte_auxiliary_driver *drv;
 397
 398         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 399                 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
 400                         return RTE_IOVA_VA;
 401         }
 402
 403         return RTE_IOVA_DC;
 404 }
 405
 406 struct rte_auxiliary_bus auxiliary_bus = {
 407         .bus = {
 408                 .scan = auxiliary_scan,
 409                 .probe = auxiliary_probe,
 410                 .find_device = auxiliary_find_device,
 411                 .plug = auxiliary_plug,
 412                 .unplug = auxiliary_unplug,
 413                 .parse = auxiliary_parse,
 414                 .dma_map = auxiliary_dma_map,
 415                 .dma_unmap = auxiliary_dma_unmap,
 416                 .get_iommu_class = auxiliary_get_iommu_class,
 417                 .dev_iterate = auxiliary_dev_iterate,
 418         },
 419         .device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
 420         .driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
 421 };
 422
 423 RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
 424 RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);