drivers/bus/auxiliary/auxiliary_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <string.h>
   6 #include <inttypes.h>
   7 #include <stdint.h>
   8 #include <stdbool.h>
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <sys/queue.h>
  12 #include <rte_errno.h>
  13 #include <rte_interrupts.h>
  14 #include <rte_log.h>
  15 #include <rte_bus.h>
  16 #include <rte_per_lcore.h>
  17 #include <rte_memory.h>
  18 #include <rte_eal.h>
  19 #include <rte_eal_paging.h>
  20 #include <rte_string_fns.h>
  21 #include <rte_common.h>
  22 #include <rte_devargs.h>
  23
  24 #include "private.h"
  25 #include "rte_bus_auxiliary.h"
  26
  27 static struct rte_devargs *
  28 auxiliary_devargs_lookup(const char *name)
  29 {
  30         struct rte_devargs *devargs;
  31
  32         RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
  33                 if (strcmp(devargs->name, name) == 0)
  34                         return devargs;
  35         }
  36         return NULL;
  37 }
  38
  39 /*
  40  * Test whether the auxiliary device exist.
  41  *
  42  * Stub for OS not supporting auxiliary bus.
  43  */
  44 __rte_weak bool
  45 auxiliary_dev_exists(const char *name)
  46 {
  47         RTE_SET_USED(name);
  48         return false;
  49 }
  50
  51 /*
  52  * Scan the devices in the auxiliary bus.
  53  *
  54  * Stub for OS not supporting auxiliary bus.
  55  */
  56 __rte_weak int
  57 auxiliary_scan(void)
  58 {
  59         return 0;
  60 }
  61
  62 /*
  63  * Update a device's devargs being scanned.
  64  */
  65 void
  66 auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
  67 {
  68         aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
  69 }
  70
  71 /*
  72  * Match the auxiliary driver and device using driver function.
  73  */
  74 bool
  75 auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
  76                 const struct rte_auxiliary_device *aux_dev)
  77 {
  78         if (aux_drv->match == NULL)
  79                 return false;
  80         return aux_drv->match(aux_dev->name);
  81 }
  82
  83 /*
  84  * Call the probe() function of the driver.
  85  */
  86 static int
  87 rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
  88                                struct rte_auxiliary_device *dev)
  89 {
  90         enum rte_iova_mode iova_mode;
  91         int ret;
  92
  93         if (drv == NULL || dev == NULL)
  94                 return -EINVAL;
  95
  96         /* Check if driver supports it. */
  97         if (!auxiliary_match(drv, dev))
  98                 /* Match of device and driver failed */
  99                 return 1;
 100
 101         /* No initialization when marked as blocked, return without error. */
 102         if (dev->device.devargs != NULL &&
 103             dev->device.devargs->policy == RTE_DEV_BLOCKED) {
 104                 AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
 105                 return -1;
 106         }
 107
 108         if (dev->device.numa_node < 0) {
 109                 AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting NUMA node to 0");
 110                 dev->device.numa_node = 0;
 111         }
 112
 113         iova_mode = rte_eal_iova_mode();
 114         if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
 115             iova_mode != RTE_IOVA_VA) {
 116                 AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
 117                               drv->driver.name);
 118                 return -EINVAL;
 119         }
 120
 121         dev->driver = drv;
 122
 123         AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)",
 124                       drv->driver.name, dev->name, dev->device.numa_node);
 125         ret = drv->probe(drv, dev);
 126         if (ret != 0)
 127                 dev->driver = NULL;
 128         else
 129                 dev->device.driver = &drv->driver;
 130
 131         return ret;
 132 }
 133
 134 /*
 135  * Call the remove() function of the driver.
 136  */
 137 static int
 138 rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
 139 {
 140         struct rte_auxiliary_driver *drv;
 141         int ret = 0;
 142
 143         if (dev == NULL)
 144                 return -EINVAL;
 145
 146         drv = dev->driver;
 147
 148         AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i",
 149                       drv->driver.name, dev->name, dev->device.numa_node);
 150
 151         if (drv->remove != NULL) {
 152                 ret = drv->remove(dev);
 153                 if (ret < 0)
 154                         return ret;
 155         }
 156
 157         /* clear driver structure */
 158         dev->driver = NULL;
 159         dev->device.driver = NULL;
 160
 161         return 0;
 162 }
 163
 164 /*
 165  * Call the probe() function of all registered drivers for the given device.
 166  * Return < 0 if initialization failed.
 167  * Return 1 if no driver is found for this device.
 168  */
 169 static int
 170 auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
 171 {
 172         struct rte_auxiliary_driver *drv;
 173         int rc;
 174
 175         if (dev == NULL)
 176                 return -EINVAL;
 177
 178         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 179                 if (!drv->match(dev->name))
 180                         continue;
 181
 182                 rc = rte_auxiliary_probe_one_driver(drv, dev);
 183                 if (rc < 0)
 184                         /* negative value is an error */
 185                         return rc;
 186                 if (rc > 0)
 187                         /* positive value means driver doesn't support it */
 188                         continue;
 189                 return 0;
 190         }
 191         return 1;
 192 }
 193
 194 /*
 195  * Scan the content of the auxiliary bus, and call the probe function for
 196  * all registered drivers to try to probe discovered devices.
 197  */
 198 static int
 199 auxiliary_probe(void)
 200 {
 201         struct rte_auxiliary_device *dev = NULL;
 202         size_t probed = 0, failed = 0;
 203         int ret = 0;
 204
 205         FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
 206                 probed++;
 207
 208                 ret = auxiliary_probe_all_drivers(dev);
 209                 if (ret < 0) {
 210                         if (ret != -EEXIST) {
 211                                 AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
 212                                               dev->name);
 213                                 rte_errno = errno;
 214                                 failed++;
 215                         }
 216                         ret = 0;
 217                 }
 218         }
 219
 220         return (probed && probed == failed) ? -1 : 0;
 221 }
 222
 223 static int
 224 auxiliary_parse(const char *name, void *addr)
 225 {
 226         struct rte_auxiliary_driver *drv = NULL;
 227         const char **out = addr;
 228
 229         /* Allow empty device name "auxiliary:" to bypass entire bus scan. */
 230         if (strlen(name) == 0)
 231                 return 0;
 232
 233         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 234                 if (drv->match(name))
 235                         break;
 236         }
 237         if (drv != NULL && addr != NULL)
 238                 *out = name;
 239         return drv != NULL ? 0 : -1;
 240 }
 241
 242 /* Register a driver */
 243 void
 244 rte_auxiliary_register(struct rte_auxiliary_driver *driver)
 245 {
 246         TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
 247         driver->bus = &auxiliary_bus;
 248 }
 249
 250 /* Unregister a driver */
 251 void
 252 rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
 253 {
 254         TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
 255         driver->bus = NULL;
 256 }
 257
 258 /* Add a device to auxiliary bus */
 259 void
 260 auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
 261 {
 262         TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
 263 }
 264
 265 /* Insert a device into a predefined position in auxiliary bus */
 266 void
 267 auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
 268                         struct rte_auxiliary_device *new_aux_dev)
 269 {
 270         TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
 271 }
 272
 273 /* Remove a device from auxiliary bus */
 274 static void
 275 rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
 276 {
 277         TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
 278 }
 279
 280 static struct rte_device *
 281 auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 282                       const void *data)
 283 {
 284         const struct rte_auxiliary_device *pstart;
 285         struct rte_auxiliary_device *adev;
 286
 287         if (start != NULL) {
 288                 pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
 289                 adev = TAILQ_NEXT(pstart, next);
 290         } else {
 291                 adev = TAILQ_FIRST(&auxiliary_bus.device_list);
 292         }
 293         while (adev != NULL) {
 294                 if (cmp(&adev->device, data) == 0)
 295                         return &adev->device;
 296                 adev = TAILQ_NEXT(adev, next);
 297         }
 298         return NULL;
 299 }
 300
 301 static int
 302 auxiliary_plug(struct rte_device *dev)
 303 {
 304         if (!auxiliary_dev_exists(dev->name))
 305                 return -ENOENT;
 306         return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
 307 }
 308
 309 static int
 310 auxiliary_unplug(struct rte_device *dev)
 311 {
 312         struct rte_auxiliary_device *adev;
 313         int ret;
 314
 315         adev = RTE_DEV_TO_AUXILIARY(dev);
 316         ret = rte_auxiliary_driver_remove_dev(adev);
 317         if (ret == 0) {
 318                 rte_auxiliary_remove_device(adev);
 319                 rte_devargs_remove(dev->devargs);
 320                 free(adev);
 321         }
 322         return ret;
 323 }
 324
 325 static int
 326 auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
 327 {
 328         struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
 329
 330         if (dev == NULL || aux_dev->driver == NULL) {
 331                 rte_errno = EINVAL;
 332                 return -1;
 333         }
 334         if (aux_dev->driver->dma_map == NULL) {
 335                 rte_errno = ENOTSUP;
 336                 return -1;
 337         }
 338         return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
 339 }
 340
 341 static int
 342 auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
 343                     size_t len)
 344 {
 345         struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
 346
 347         if (dev == NULL || aux_dev->driver == NULL) {
 348                 rte_errno = EINVAL;
 349                 return -1;
 350         }
 351         if (aux_dev->driver->dma_unmap == NULL) {
 352                 rte_errno = ENOTSUP;
 353                 return -1;
 354         }
 355         return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
 356 }
 357
 358 bool
 359 auxiliary_is_ignored_device(const char *name)
 360 {
 361         struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
 362
 363         switch (auxiliary_bus.bus.conf.scan_mode) {
 364         case RTE_BUS_SCAN_ALLOWLIST:
 365                 if (devargs && devargs->policy == RTE_DEV_ALLOWED)
 366                         return false;
 367                 break;
 368         case RTE_BUS_SCAN_UNDEFINED:
 369         case RTE_BUS_SCAN_BLOCKLIST:
 370                 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
 371                         return false;
 372                 break;
 373         }
 374         return true;
 375 }
 376
 377 static enum rte_iova_mode
 378 auxiliary_get_iommu_class(void)
 379 {
 380         const struct rte_auxiliary_driver *drv;
 381
 382         FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
 383                 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
 384                         return RTE_IOVA_VA;
 385         }
 386
 387         return RTE_IOVA_DC;
 388 }
 389
 390 struct rte_auxiliary_bus auxiliary_bus = {
 391         .bus = {
 392                 .scan = auxiliary_scan,
 393                 .probe = auxiliary_probe,
 394                 .find_device = auxiliary_find_device,
 395                 .plug = auxiliary_plug,
 396                 .unplug = auxiliary_unplug,
 397                 .parse = auxiliary_parse,
 398                 .dma_map = auxiliary_dma_map,
 399                 .dma_unmap = auxiliary_dma_unmap,
 400                 .get_iommu_class = auxiliary_get_iommu_class,
 401                 .dev_iterate = auxiliary_dev_iterate,
 402         },
 403         .device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
 404         .driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
 405 };
 406
 407 RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
 408 RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);