1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
11 #include <sys/queue.h>
12 #include <rte_errno.h>
13 #include <rte_interrupts.h>
16 #include <rte_per_lcore.h>
17 #include <rte_memory.h>
19 #include <rte_eal_paging.h>
20 #include <rte_string_fns.h>
21 #include <rte_common.h>
22 #include <rte_devargs.h>
25 #include "rte_bus_auxiliary.h"
27 static struct rte_devargs *
28 auxiliary_devargs_lookup(const char *name)
30 struct rte_devargs *devargs;
32 RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
33 if (strcmp(devargs->name, name) == 0)
40 * Test whether the auxiliary device exist.
42 * Stub for OS not supporting auxiliary bus.
45 auxiliary_dev_exists(const char *name)
52 * Scan the devices in the auxiliary bus.
54 * Stub for OS not supporting auxiliary bus.
63 * Update a device's devargs being scanned.
66 auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
68 aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
72 * Match the auxiliary driver and device using driver function.
75 auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
76 const struct rte_auxiliary_device *aux_dev)
78 if (aux_drv->match == NULL)
80 return aux_drv->match(aux_dev->name);
84 * Call the probe() function of the driver.
87 rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
88 struct rte_auxiliary_device *dev)
90 enum rte_iova_mode iova_mode;
93 if (drv == NULL || dev == NULL)
96 /* Check if driver supports it. */
97 if (!auxiliary_match(drv, dev))
98 /* Match of device and driver failed */
101 /* No initialization when marked as blocked, return without error. */
102 if (dev->device.devargs != NULL &&
103 dev->device.devargs->policy == RTE_DEV_BLOCKED) {
104 AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
108 if (dev->device.numa_node < 0) {
109 AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting NUMA node to 0");
110 dev->device.numa_node = 0;
113 iova_mode = rte_eal_iova_mode();
114 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
115 iova_mode != RTE_IOVA_VA) {
116 AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
123 AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)",
124 drv->driver.name, dev->name, dev->device.numa_node);
125 ret = drv->probe(drv, dev);
129 dev->device.driver = &drv->driver;
135 * Call the remove() function of the driver.
138 rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
140 struct rte_auxiliary_driver *drv;
148 AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i",
149 drv->driver.name, dev->name, dev->device.numa_node);
151 if (drv->remove != NULL) {
152 ret = drv->remove(dev);
157 /* clear driver structure */
159 dev->device.driver = NULL;
165 * Call the probe() function of all registered drivers for the given device.
166 * Return < 0 if initialization failed.
167 * Return 1 if no driver is found for this device.
170 auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
172 struct rte_auxiliary_driver *drv;
178 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
179 if (!drv->match(dev->name))
182 rc = rte_auxiliary_probe_one_driver(drv, dev);
184 /* negative value is an error */
187 /* positive value means driver doesn't support it */
195 * Scan the content of the auxiliary bus, and call the probe function for
196 * all registered drivers to try to probe discovered devices.
199 auxiliary_probe(void)
201 struct rte_auxiliary_device *dev = NULL;
202 size_t probed = 0, failed = 0;
205 FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
208 ret = auxiliary_probe_all_drivers(dev);
210 if (ret != -EEXIST) {
211 AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
220 return (probed && probed == failed) ? -1 : 0;
224 auxiliary_parse(const char *name, void *addr)
226 struct rte_auxiliary_driver *drv = NULL;
227 const char **out = addr;
229 /* Allow empty device name "auxiliary:" to bypass entire bus scan. */
230 if (strlen(name) == 0)
233 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
234 if (drv->match(name))
237 if (drv != NULL && addr != NULL)
239 return drv != NULL ? 0 : -1;
242 /* Register a driver */
244 rte_auxiliary_register(struct rte_auxiliary_driver *driver)
246 TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
247 driver->bus = &auxiliary_bus;
250 /* Unregister a driver */
252 rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
254 TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
258 /* Add a device to auxiliary bus */
260 auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
262 TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
265 /* Insert a device into a predefined position in auxiliary bus */
267 auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
268 struct rte_auxiliary_device *new_aux_dev)
270 TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
273 /* Remove a device from auxiliary bus */
275 rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
277 TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
280 static struct rte_device *
281 auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
284 const struct rte_auxiliary_device *pstart;
285 struct rte_auxiliary_device *adev;
288 pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
289 adev = TAILQ_NEXT(pstart, next);
291 adev = TAILQ_FIRST(&auxiliary_bus.device_list);
293 while (adev != NULL) {
294 if (cmp(&adev->device, data) == 0)
295 return &adev->device;
296 adev = TAILQ_NEXT(adev, next);
302 auxiliary_plug(struct rte_device *dev)
304 if (!auxiliary_dev_exists(dev->name))
306 return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
310 auxiliary_unplug(struct rte_device *dev)
312 struct rte_auxiliary_device *adev;
315 adev = RTE_DEV_TO_AUXILIARY(dev);
316 ret = rte_auxiliary_driver_remove_dev(adev);
318 rte_auxiliary_remove_device(adev);
319 rte_devargs_remove(dev->devargs);
326 auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
328 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
330 if (dev == NULL || aux_dev->driver == NULL) {
334 if (aux_dev->driver->dma_map == NULL) {
338 return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
342 auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
345 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
347 if (dev == NULL || aux_dev->driver == NULL) {
351 if (aux_dev->driver->dma_unmap == NULL) {
355 return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
359 auxiliary_is_ignored_device(const char *name)
361 struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
363 switch (auxiliary_bus.bus.conf.scan_mode) {
364 case RTE_BUS_SCAN_ALLOWLIST:
365 if (devargs && devargs->policy == RTE_DEV_ALLOWED)
368 case RTE_BUS_SCAN_UNDEFINED:
369 case RTE_BUS_SCAN_BLOCKLIST:
370 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
377 static enum rte_iova_mode
378 auxiliary_get_iommu_class(void)
380 const struct rte_auxiliary_driver *drv;
382 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
383 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
390 struct rte_auxiliary_bus auxiliary_bus = {
392 .scan = auxiliary_scan,
393 .probe = auxiliary_probe,
394 .find_device = auxiliary_find_device,
395 .plug = auxiliary_plug,
396 .unplug = auxiliary_unplug,
397 .parse = auxiliary_parse,
398 .dma_map = auxiliary_dma_map,
399 .dma_unmap = auxiliary_dma_unmap,
400 .get_iommu_class = auxiliary_get_iommu_class,
401 .dev_iterate = auxiliary_dev_iterate,
403 .device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
404 .driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
407 RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
408 RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);