1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
11 #include <sys/queue.h>
12 #include <rte_errno.h>
13 #include <rte_interrupts.h>
16 #include <rte_per_lcore.h>
17 #include <rte_memory.h>
19 #include <rte_eal_paging.h>
20 #include <rte_lcore.h>
21 #include <rte_string_fns.h>
22 #include <rte_common.h>
23 #include <rte_devargs.h>
26 #include "rte_bus_auxiliary.h"
28 static struct rte_devargs *
29 auxiliary_devargs_lookup(const char *name)
31 struct rte_devargs *devargs;
33 RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
34 if (strcmp(devargs->name, name) == 0)
41 * Test whether the auxiliary device exist.
43 * Stub for OS not supporting auxiliary bus.
46 auxiliary_dev_exists(const char *name)
53 * Scan the devices in the auxiliary bus.
55 * Stub for OS not supporting auxiliary bus.
64 * Update a device's devargs being scanned.
67 auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
69 aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
73 * Match the auxiliary driver and device using driver function.
76 auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
77 const struct rte_auxiliary_device *aux_dev)
79 if (aux_drv->match == NULL)
81 return aux_drv->match(aux_dev->name);
85 * Call the probe() function of the driver.
88 rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
89 struct rte_auxiliary_device *dev)
91 enum rte_iova_mode iova_mode;
94 if (drv == NULL || dev == NULL)
97 /* Check if driver supports it. */
98 if (!auxiliary_match(drv, dev))
99 /* Match of device and driver failed */
102 /* No initialization when marked as blocked, return without error. */
103 if (dev->device.devargs != NULL &&
104 dev->device.devargs->policy == RTE_DEV_BLOCKED) {
105 AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
109 if (dev->device.numa_node < 0) {
110 if (rte_socket_count() > 1)
111 AUXILIARY_LOG(INFO, "Device %s is not NUMA-aware, defaulting socket to 0",
113 dev->device.numa_node = 0;
116 iova_mode = rte_eal_iova_mode();
117 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
118 iova_mode != RTE_IOVA_VA) {
119 AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
126 AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)",
127 drv->driver.name, dev->name, dev->device.numa_node);
128 ret = drv->probe(drv, dev);
132 dev->device.driver = &drv->driver;
138 * Call the remove() function of the driver.
141 rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
143 struct rte_auxiliary_driver *drv;
151 AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i",
152 drv->driver.name, dev->name, dev->device.numa_node);
154 if (drv->remove != NULL) {
155 ret = drv->remove(dev);
160 /* clear driver structure */
162 dev->device.driver = NULL;
168 * Call the probe() function of all registered drivers for the given device.
169 * Return < 0 if initialization failed.
170 * Return 1 if no driver is found for this device.
173 auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
175 struct rte_auxiliary_driver *drv;
181 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
182 if (!drv->match(dev->name))
185 rc = rte_auxiliary_probe_one_driver(drv, dev);
187 /* negative value is an error */
190 /* positive value means driver doesn't support it */
198 * Scan the content of the auxiliary bus, and call the probe function for
199 * all registered drivers to try to probe discovered devices.
202 auxiliary_probe(void)
204 struct rte_auxiliary_device *dev = NULL;
205 size_t probed = 0, failed = 0;
208 FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
211 ret = auxiliary_probe_all_drivers(dev);
213 if (ret != -EEXIST) {
214 AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
223 return (probed && probed == failed) ? -1 : 0;
227 auxiliary_parse(const char *name, void *addr)
229 struct rte_auxiliary_driver *drv = NULL;
230 const char **out = addr;
232 /* Allow empty device name "auxiliary:" to bypass entire bus scan. */
233 if (strlen(name) == 0)
236 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
237 if (drv->match(name))
240 if (drv != NULL && addr != NULL)
242 return drv != NULL ? 0 : -1;
245 /* Register a driver */
247 rte_auxiliary_register(struct rte_auxiliary_driver *driver)
249 TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
250 driver->bus = &auxiliary_bus;
253 /* Unregister a driver */
255 rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
257 TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
261 /* Add a device to auxiliary bus */
263 auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
265 TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
268 /* Insert a device into a predefined position in auxiliary bus */
270 auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
271 struct rte_auxiliary_device *new_aux_dev)
273 TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
276 /* Remove a device from auxiliary bus */
278 rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
280 TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
283 static struct rte_device *
284 auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
287 const struct rte_auxiliary_device *pstart;
288 struct rte_auxiliary_device *adev;
291 pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
292 adev = TAILQ_NEXT(pstart, next);
294 adev = TAILQ_FIRST(&auxiliary_bus.device_list);
296 while (adev != NULL) {
297 if (cmp(&adev->device, data) == 0)
298 return &adev->device;
299 adev = TAILQ_NEXT(adev, next);
305 auxiliary_plug(struct rte_device *dev)
307 if (!auxiliary_dev_exists(dev->name))
309 return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
313 auxiliary_unplug(struct rte_device *dev)
315 struct rte_auxiliary_device *adev;
318 adev = RTE_DEV_TO_AUXILIARY(dev);
319 ret = rte_auxiliary_driver_remove_dev(adev);
321 rte_auxiliary_remove_device(adev);
322 rte_devargs_remove(dev->devargs);
329 auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
331 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
333 if (dev == NULL || aux_dev->driver == NULL) {
337 if (aux_dev->driver->dma_map == NULL) {
341 return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
345 auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
348 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
350 if (dev == NULL || aux_dev->driver == NULL) {
354 if (aux_dev->driver->dma_unmap == NULL) {
358 return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
362 auxiliary_is_ignored_device(const char *name)
364 struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
366 switch (auxiliary_bus.bus.conf.scan_mode) {
367 case RTE_BUS_SCAN_ALLOWLIST:
368 if (devargs && devargs->policy == RTE_DEV_ALLOWED)
371 case RTE_BUS_SCAN_UNDEFINED:
372 case RTE_BUS_SCAN_BLOCKLIST:
373 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
380 static enum rte_iova_mode
381 auxiliary_get_iommu_class(void)
383 const struct rte_auxiliary_driver *drv;
385 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
386 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
393 struct rte_auxiliary_bus auxiliary_bus = {
395 .scan = auxiliary_scan,
396 .probe = auxiliary_probe,
397 .find_device = auxiliary_find_device,
398 .plug = auxiliary_plug,
399 .unplug = auxiliary_unplug,
400 .parse = auxiliary_parse,
401 .dma_map = auxiliary_dma_map,
402 .dma_unmap = auxiliary_dma_unmap,
403 .get_iommu_class = auxiliary_get_iommu_class,
404 .dev_iterate = auxiliary_dev_iterate,
406 .device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
407 .driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
410 RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
411 RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);