1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies Ltd
6 #include <rte_malloc.h>
7 #include "mlx5_common_utils.h"
8 #include "mlx5_common_pci.h"
10 struct mlx5_pci_device {
11 struct rte_pci_device *pci_dev;
12 TAILQ_ENTRY(mlx5_pci_device) next;
13 uint32_t classes_loaded;
16 /* Head of list of drivers. */
17 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list =
18 TAILQ_HEAD_INITIALIZER(drv_list);
20 /* Head of mlx5 pci devices. */
21 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
22 TAILQ_HEAD_INITIALIZER(devices_list);
26 unsigned int driver_class;
28 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA },
29 { .name = "net", .driver_class = MLX5_CLASS_NET },
30 { .name = "regex", .driver_class = MLX5_CLASS_REGEX },
33 static const unsigned int mlx5_class_combinations[] = {
37 MLX5_CLASS_NET | MLX5_CLASS_REGEX,
38 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
39 /* New class combination should be added here. */
43 class_name_to_value(const char *class_name)
47 for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
48 if (strcmp(class_name, mlx5_classes[i].name) == 0)
49 return mlx5_classes[i].driver_class;
54 static struct mlx5_pci_driver *
55 driver_get(uint32_t class)
57 struct mlx5_pci_driver *driver;
59 TAILQ_FOREACH(driver, &drv_list, next) {
60 if (driver->driver_class == class)
67 bus_cmdline_options_handler(__rte_unused const char *key,
68 const char *class_names, void *opaque)
77 nstr = strdup(class_names);
84 /* Extract each individual class name. Multiple
85 * class key,value is supplied as class=net:vdpa:foo:bar.
87 found = strsep(&nstr, ":");
90 /* Check if its a valid class. */
91 class_val = class_name_to_value(found);
101 DRV_LOG(ERR, "Invalid mlx5 class options %s."
102 " Maybe typo in device class argument setting?",
108 parse_class_options(const struct rte_devargs *devargs)
110 const char *key = MLX5_CLASS_ARG_NAME;
111 struct rte_kvargs *kvlist;
116 kvlist = rte_kvargs_parse(devargs->args, NULL);
119 if (rte_kvargs_count(kvlist, key))
120 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler,
122 rte_kvargs_free(kvlist);
127 mlx5_bus_match(const struct mlx5_pci_driver *drv,
128 const struct rte_pci_device *pci_dev)
130 const struct rte_pci_id *id_table;
132 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0;
134 /* Check if device's ids match the class driver's ids. */
135 if (id_table->vendor_id != pci_dev->id.vendor_id &&
136 id_table->vendor_id != PCI_ANY_ID)
138 if (id_table->device_id != pci_dev->id.device_id &&
139 id_table->device_id != PCI_ANY_ID)
141 if (id_table->subsystem_vendor_id !=
142 pci_dev->id.subsystem_vendor_id &&
143 id_table->subsystem_vendor_id != PCI_ANY_ID)
145 if (id_table->subsystem_device_id !=
146 pci_dev->id.subsystem_device_id &&
147 id_table->subsystem_device_id != PCI_ANY_ID)
149 if (id_table->class_id != pci_dev->id.class_id &&
150 id_table->class_id != RTE_CLASS_ANY_ID)
158 is_valid_class_combination(uint32_t user_classes)
162 /* Verify if user specified valid supported combination. */
163 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) {
164 if (mlx5_class_combinations[i] == user_classes)
167 /* Not found any valid class combination. */
171 static struct mlx5_pci_device *
172 pci_to_mlx5_device(const struct rte_pci_device *pci_dev)
174 struct mlx5_pci_device *dev;
176 TAILQ_FOREACH(dev, &devices_list, next) {
177 if (dev->pci_dev == pci_dev)
184 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class)
186 return (device->classes_loaded & class) ? true : false;
190 dev_release(struct mlx5_pci_device *dev)
192 TAILQ_REMOVE(&devices_list, dev, next);
197 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes)
199 struct mlx5_pci_driver *driver;
200 int local_ret = -ENODEV;
204 enabled_classes &= dev->classes_loaded;
205 while (enabled_classes) {
206 driver = driver_get(RTE_BIT64(i));
208 local_ret = driver->pci_driver.remove(dev->pci_dev);
210 dev->classes_loaded &= ~RTE_BIT64(i);
214 enabled_classes &= ~RTE_BIT64(i);
223 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv,
224 struct rte_pci_device *pci_dev, uint32_t user_classes)
226 struct mlx5_pci_driver *driver;
227 uint32_t enabled_classes = 0;
231 TAILQ_FOREACH(driver, &drv_list, next) {
232 if ((driver->driver_class & user_classes) == 0)
234 if (!mlx5_bus_match(driver, pci_dev))
236 already_loaded = dev->classes_loaded & driver->driver_class;
237 if (already_loaded &&
238 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
239 DRV_LOG(ERR, "Device %s is already probed\n",
240 pci_dev->device.name);
244 ret = driver->pci_driver.probe(pci_drv, pci_dev);
246 DRV_LOG(ERR, "Failed to load driver = %s.\n",
247 driver->pci_driver.driver.name);
250 enabled_classes |= driver->driver_class;
252 dev->classes_loaded |= enabled_classes;
255 /* Only unload drivers which are enabled which were enabled
256 * in this probe instance.
258 drivers_remove(dev, enabled_classes);
263 * DPDK callback to register to probe multiple drivers for a PCI device.
266 * PCI driver structure.
268 * PCI device information.
271 * 0 on success, a negative errno value otherwise and rte_errno is set.
274 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
275 struct rte_pci_device *pci_dev)
277 struct mlx5_pci_device *dev;
278 uint32_t user_classes = 0;
279 bool new_device = false;
282 ret = parse_class_options(pci_dev->device.devargs);
287 /* Validate combination here. */
288 ret = is_valid_class_combination(user_classes);
290 DRV_LOG(ERR, "Unsupported mlx5 classes supplied.");
294 /* Default to net class. */
295 user_classes = MLX5_CLASS_NET;
297 dev = pci_to_mlx5_device(pci_dev);
299 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0);
302 dev->pci_dev = pci_dev;
303 TAILQ_INSERT_HEAD(&devices_list, dev, next);
306 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes);
317 * DPDK callback to remove one or more drivers for a PCI device.
319 * This function removes all drivers probed for a given PCI device.
322 * Pointer to the PCI device.
325 * 0 on success, the function cannot fail.
328 mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
330 struct mlx5_pci_device *dev;
333 dev = pci_to_mlx5_device(pci_dev);
336 /* Matching device found, cleanup and unload drivers. */
337 ret = drivers_remove(dev, dev->classes_loaded);
344 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
345 uint64_t iova, size_t len)
347 struct mlx5_pci_driver *driver = NULL;
348 struct mlx5_pci_driver *temp;
349 struct mlx5_pci_device *dev;
352 dev = pci_to_mlx5_device(pci_dev);
355 TAILQ_FOREACH(driver, &drv_list, next) {
356 if (device_class_enabled(dev, driver->driver_class) &&
357 driver->pci_driver.dma_map) {
358 ret = driver->pci_driver.dma_map(pci_dev, addr,
366 TAILQ_FOREACH(temp, &drv_list, next) {
369 if (device_class_enabled(dev, temp->driver_class) &&
370 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap)
371 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len);
377 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
378 uint64_t iova, size_t len)
380 struct mlx5_pci_driver *driver;
381 struct mlx5_pci_device *dev;
382 int local_ret = -EINVAL;
385 dev = pci_to_mlx5_device(pci_dev);
389 /* There is no unmap error recovery in current implementation. */
390 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) {
391 if (device_class_enabled(dev, driver->driver_class) &&
392 driver->pci_driver.dma_unmap) {
393 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr,
395 if (local_ret && (ret == 0))
404 /* PCI ID table is build dynamically based on registered mlx5 drivers. */
405 static struct rte_pci_id *mlx5_pci_id_table;
407 static struct rte_pci_driver mlx5_pci_driver = {
411 .probe = mlx5_common_pci_probe,
412 .remove = mlx5_common_pci_remove,
413 .dma_map = mlx5_common_pci_dma_map,
414 .dma_unmap = mlx5_common_pci_dma_unmap,
418 pci_id_table_size_get(const struct rte_pci_id *id_table)
422 for (; id_table->vendor_id != 0; id_table++)
428 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table,
431 int current_size = next_idx - 1;
434 for (i = 0; i < current_size; i++) {
435 if (id->device_id == table[i].device_id &&
436 id->vendor_id == table[i].vendor_id &&
437 id->subsystem_vendor_id == table[i].subsystem_vendor_id &&
438 id->subsystem_device_id == table[i].subsystem_device_id)
445 pci_id_insert(struct rte_pci_id *new_table, int *next_idx,
446 const struct rte_pci_id *id_table)
448 /* Traverse the id_table, check if entry exists in new_table;
449 * Add non duplicate entries to new table.
451 for (; id_table->vendor_id != 0; id_table++) {
452 if (!pci_id_exists(id_table, new_table, *next_idx)) {
453 /* New entry; add to the table. */
454 new_table[*next_idx] = *id_table;
461 pci_ids_table_update(const struct rte_pci_id *driver_id_table)
463 const struct rte_pci_id *id_iter;
464 struct rte_pci_id *updated_table;
465 struct rte_pci_id *old_table;
469 old_table = mlx5_pci_id_table;
471 num_ids = pci_id_table_size_get(old_table);
472 num_ids += pci_id_table_size_get(driver_id_table);
473 /* Increase size by one for the termination entry of vendor_id = 0. */
475 updated_table = calloc(num_ids, sizeof(*updated_table));
478 if (TAILQ_EMPTY(&drv_list)) {
479 /* Copy the first driver's ID table. */
480 for (id_iter = driver_id_table; id_iter->vendor_id != 0;
482 updated_table[i] = *id_iter;
484 /* First copy existing table entries. */
485 for (id_iter = old_table; id_iter->vendor_id != 0;
487 updated_table[i] = *id_iter;
488 /* New id to be added at the end of current ID table. */
489 pci_id_insert(updated_table, &i, driver_id_table);
491 /* Terminate table with empty entry. */
492 updated_table[i].vendor_id = 0;
493 mlx5_pci_driver.id_table = updated_table;
494 mlx5_pci_id_table = updated_table;
501 mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
505 ret = pci_ids_table_update(driver->pci_driver.id_table);
508 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags;
509 TAILQ_INSERT_TAIL(&drv_list, driver, next);
512 void mlx5_common_pci_init(void)
514 const struct rte_pci_id empty_table[] = {
520 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD
521 * including this one can register the PCI table first. If any other
522 * PMD(s) have registered the PCI ID table, No need to register an empty
525 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table))
527 rte_pci_register(&mlx5_pci_driver);
530 RTE_FINI(mlx5_common_pci_finish)
532 if (mlx5_pci_id_table != NULL) {
533 /* Constructor doesn't register with PCI bus if it failed
534 * to build the table.
536 rte_pci_unregister(&mlx5_pci_driver);
537 free(mlx5_pci_id_table);
540 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__);