1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
11 #include <sys/queue.h>
12 #include <rte_errno.h>
13 #include <rte_interrupts.h>
16 #include <rte_per_lcore.h>
17 #include <rte_memory.h>
19 #include <rte_eal_paging.h>
20 #include <rte_lcore.h>
21 #include <rte_string_fns.h>
22 #include <rte_common.h>
23 #include <rte_devargs.h>
26 #include "rte_bus_auxiliary.h"
28 static struct rte_devargs *
29 auxiliary_devargs_lookup(const char *name)
31 struct rte_devargs *devargs;
33 RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
34 if (strcmp(devargs->name, name) == 0)
41 * Test whether the auxiliary device exist.
43 * Stub for OS not supporting auxiliary bus.
46 auxiliary_dev_exists(const char *name)
53 * Scan the devices in the auxiliary bus.
55 * Stub for OS not supporting auxiliary bus.
64 * Update a device's devargs being scanned.
67 auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
69 aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
73 * Match the auxiliary driver and device using driver function.
76 auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
77 const struct rte_auxiliary_device *aux_dev)
79 if (aux_drv->match == NULL)
81 return aux_drv->match(aux_dev->name);
85 * Call the probe() function of the driver.
88 rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
89 struct rte_auxiliary_device *dev)
91 enum rte_iova_mode iova_mode;
94 if (drv == NULL || dev == NULL)
97 /* Check if driver supports it. */
98 if (!auxiliary_match(drv, dev))
99 /* Match of device and driver failed */
102 /* No initialization when marked as blocked, return without error. */
103 if (dev->device.devargs != NULL &&
104 dev->device.devargs->policy == RTE_DEV_BLOCKED) {
105 AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
109 if (dev->device.numa_node < 0) {
110 if (rte_socket_count() > 1)
111 AUXILIARY_LOG(INFO, "Device %s is not NUMA-aware, defaulting socket to 0",
113 dev->device.numa_node = 0;
116 iova_mode = rte_eal_iova_mode();
117 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
118 iova_mode != RTE_IOVA_VA) {
119 AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
124 /* Allocate interrupt instance */
126 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE);
127 if (dev->intr_handle == NULL) {
128 AUXILIARY_LOG(ERR, "Could not allocate interrupt instance for device %s",
135 AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)",
136 drv->driver.name, dev->name, dev->device.numa_node);
137 ret = drv->probe(drv, dev);
140 rte_intr_instance_free(dev->intr_handle);
141 dev->intr_handle = NULL;
143 dev->device.driver = &drv->driver;
150 * Call the remove() function of the driver.
153 rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
155 struct rte_auxiliary_driver *drv;
163 AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i",
164 drv->driver.name, dev->name, dev->device.numa_node);
166 if (drv->remove != NULL) {
167 ret = drv->remove(dev);
172 /* clear driver structure */
174 dev->device.driver = NULL;
180 * Call the probe() function of all registered drivers for the given device.
181 * Return < 0 if initialization failed.
182 * Return 1 if no driver is found for this device.
185 auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
187 struct rte_auxiliary_driver *drv;
193 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
194 if (!drv->match(dev->name))
197 rc = rte_auxiliary_probe_one_driver(drv, dev);
199 /* negative value is an error */
202 /* positive value means driver doesn't support it */
210 * Scan the content of the auxiliary bus, and call the probe function for
211 * all registered drivers to try to probe discovered devices.
214 auxiliary_probe(void)
216 struct rte_auxiliary_device *dev = NULL;
217 size_t probed = 0, failed = 0;
220 FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
223 ret = auxiliary_probe_all_drivers(dev);
225 if (ret != -EEXIST) {
226 AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
235 return (probed && probed == failed) ? -1 : 0;
239 auxiliary_parse(const char *name, void *addr)
241 struct rte_auxiliary_driver *drv = NULL;
242 const char **out = addr;
244 /* Allow empty device name "auxiliary:" to bypass entire bus scan. */
245 if (strlen(name) == 0)
248 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
249 if (drv->match(name))
252 if (drv != NULL && addr != NULL)
254 return drv != NULL ? 0 : -1;
257 /* Register a driver */
259 rte_auxiliary_register(struct rte_auxiliary_driver *driver)
261 TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
262 driver->bus = &auxiliary_bus;
265 /* Unregister a driver */
267 rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
269 TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
273 /* Add a device to auxiliary bus */
275 auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
277 TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
280 /* Insert a device into a predefined position in auxiliary bus */
282 auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
283 struct rte_auxiliary_device *new_aux_dev)
285 TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
288 /* Remove a device from auxiliary bus */
290 rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
292 TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
295 static struct rte_device *
296 auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
299 const struct rte_auxiliary_device *pstart;
300 struct rte_auxiliary_device *adev;
303 pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
304 adev = TAILQ_NEXT(pstart, next);
306 adev = TAILQ_FIRST(&auxiliary_bus.device_list);
308 while (adev != NULL) {
309 if (cmp(&adev->device, data) == 0)
310 return &adev->device;
311 adev = TAILQ_NEXT(adev, next);
317 auxiliary_plug(struct rte_device *dev)
319 if (!auxiliary_dev_exists(dev->name))
321 return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
325 auxiliary_unplug(struct rte_device *dev)
327 struct rte_auxiliary_device *adev;
330 adev = RTE_DEV_TO_AUXILIARY(dev);
331 ret = rte_auxiliary_driver_remove_dev(adev);
333 rte_auxiliary_remove_device(adev);
334 rte_devargs_remove(dev->devargs);
335 rte_intr_instance_free(adev->intr_handle);
342 auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
344 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
346 if (dev == NULL || aux_dev->driver == NULL) {
350 if (aux_dev->driver->dma_map == NULL) {
354 return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
358 auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
361 struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
363 if (dev == NULL || aux_dev->driver == NULL) {
367 if (aux_dev->driver->dma_unmap == NULL) {
371 return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
375 auxiliary_is_ignored_device(const char *name)
377 struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
379 switch (auxiliary_bus.bus.conf.scan_mode) {
380 case RTE_BUS_SCAN_ALLOWLIST:
381 if (devargs && devargs->policy == RTE_DEV_ALLOWED)
384 case RTE_BUS_SCAN_UNDEFINED:
385 case RTE_BUS_SCAN_BLOCKLIST:
386 if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
393 static enum rte_iova_mode
394 auxiliary_get_iommu_class(void)
396 const struct rte_auxiliary_driver *drv;
398 FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
399 if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
406 struct rte_auxiliary_bus auxiliary_bus = {
408 .scan = auxiliary_scan,
409 .probe = auxiliary_probe,
410 .find_device = auxiliary_find_device,
411 .plug = auxiliary_plug,
412 .unplug = auxiliary_unplug,
413 .parse = auxiliary_parse,
414 .dma_map = auxiliary_dma_map,
415 .dma_unmap = auxiliary_dma_unmap,
416 .get_iommu_class = auxiliary_get_iommu_class,
417 .dev_iterate = auxiliary_dev_iterate,
419 .device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
420 .driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
423 RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
424 RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);