1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies Ltd
6 #include <rte_malloc.h>
9 #include "mlx5_common_log.h"
10 #include "mlx5_common_pci.h"
12 struct mlx5_pci_device {
13 struct rte_pci_device *pci_dev;
14 TAILQ_ENTRY(mlx5_pci_device) next;
15 uint32_t classes_loaded;
18 /* Head of list of drivers. */
19 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list =
20 TAILQ_HEAD_INITIALIZER(drv_list);
22 /* Head of mlx5 pci devices. */
23 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
24 TAILQ_HEAD_INITIALIZER(devices_list);
28 unsigned int driver_class;
30 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA },
31 { .name = "net", .driver_class = MLX5_CLASS_NET },
32 { .name = "regex", .driver_class = MLX5_CLASS_REGEX },
33 { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
36 static const unsigned int mlx5_class_combinations[] = {
41 MLX5_CLASS_NET | MLX5_CLASS_REGEX,
42 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
43 MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
44 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
45 MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
46 MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
47 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
48 /* New class combination should be added here. */
52 class_name_to_value(const char *class_name)
56 for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
57 if (strcmp(class_name, mlx5_classes[i].name) == 0)
58 return mlx5_classes[i].driver_class;
63 static struct mlx5_pci_driver *
64 driver_get(uint32_t class)
66 struct mlx5_pci_driver *driver;
68 TAILQ_FOREACH(driver, &drv_list, next) {
69 if (driver->driver_class == class)
76 bus_cmdline_options_handler(__rte_unused const char *key,
77 const char *class_names, void *opaque)
87 nstr = strdup(class_names);
93 found = strtok_r(nstr, ":", &refstr);
97 /* Extract each individual class name. Multiple
98 * class key,value is supplied as class=net:vdpa:foo:bar.
100 class_val = class_name_to_value(found);
101 /* Check if its a valid class. */
107 found = strtok_r(NULL, ":", &refstr);
112 DRV_LOG(ERR, "Invalid mlx5 class options %s."
113 " Maybe typo in device class argument setting?",
119 parse_class_options(const struct rte_devargs *devargs)
121 const char *key = RTE_DEVARGS_KEY_CLASS;
122 struct rte_kvargs *kvlist;
127 kvlist = rte_kvargs_parse(devargs->args, NULL);
130 if (rte_kvargs_count(kvlist, key))
131 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler,
133 rte_kvargs_free(kvlist);
138 mlx5_bus_match(const struct mlx5_pci_driver *drv,
139 const struct rte_pci_device *pci_dev)
141 const struct rte_pci_id *id_table;
143 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0;
145 /* Check if device's ids match the class driver's ids. */
146 if (id_table->vendor_id != pci_dev->id.vendor_id &&
147 id_table->vendor_id != RTE_PCI_ANY_ID)
149 if (id_table->device_id != pci_dev->id.device_id &&
150 id_table->device_id != RTE_PCI_ANY_ID)
152 if (id_table->subsystem_vendor_id !=
153 pci_dev->id.subsystem_vendor_id &&
154 id_table->subsystem_vendor_id != RTE_PCI_ANY_ID)
156 if (id_table->subsystem_device_id !=
157 pci_dev->id.subsystem_device_id &&
158 id_table->subsystem_device_id != RTE_PCI_ANY_ID)
160 if (id_table->class_id != pci_dev->id.class_id &&
161 id_table->class_id != RTE_CLASS_ANY_ID)
169 is_valid_class_combination(uint32_t user_classes)
173 /* Verify if user specified valid supported combination. */
174 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) {
175 if (mlx5_class_combinations[i] == user_classes)
178 /* Not found any valid class combination. */
182 static struct mlx5_pci_device *
183 pci_to_mlx5_device(const struct rte_pci_device *pci_dev)
185 struct mlx5_pci_device *dev;
187 TAILQ_FOREACH(dev, &devices_list, next) {
188 if (dev->pci_dev == pci_dev)
195 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class)
197 return (device->classes_loaded & class) ? true : false;
201 dev_release(struct mlx5_pci_device *dev)
203 TAILQ_REMOVE(&devices_list, dev, next);
208 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes)
210 struct mlx5_pci_driver *driver;
211 int local_ret = -ENODEV;
215 enabled_classes &= dev->classes_loaded;
216 while (enabled_classes) {
217 driver = driver_get(RTE_BIT64(i));
219 local_ret = driver->pci_driver.remove(dev->pci_dev);
221 dev->classes_loaded &= ~RTE_BIT64(i);
225 enabled_classes &= ~RTE_BIT64(i);
234 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv,
235 struct rte_pci_device *pci_dev, uint32_t user_classes)
237 struct mlx5_pci_driver *driver;
238 uint32_t enabled_classes = 0;
242 TAILQ_FOREACH(driver, &drv_list, next) {
243 if ((driver->driver_class & user_classes) == 0)
245 if (!mlx5_bus_match(driver, pci_dev))
247 already_loaded = dev->classes_loaded & driver->driver_class;
248 if (already_loaded &&
249 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
250 DRV_LOG(ERR, "Device %s is already probed",
251 pci_dev->device.name);
255 ret = driver->pci_driver.probe(pci_drv, pci_dev);
257 DRV_LOG(ERR, "Failed to load driver %s",
258 driver->pci_driver.driver.name);
261 enabled_classes |= driver->driver_class;
263 dev->classes_loaded |= enabled_classes;
266 /* Only unload drivers which are enabled which were enabled
267 * in this probe instance.
269 drivers_remove(dev, enabled_classes);
274 * DPDK callback to register to probe multiple drivers for a PCI device.
277 * PCI driver structure.
279 * PCI device information.
282 * 0 on success, a negative errno value otherwise and rte_errno is set.
285 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
286 struct rte_pci_device *pci_dev)
288 struct mlx5_pci_device *dev;
289 uint32_t user_classes = 0;
290 bool new_device = false;
293 ret = parse_class_options(pci_dev->device.devargs);
298 /* Validate combination here. */
299 ret = is_valid_class_combination(user_classes);
301 DRV_LOG(ERR, "Unsupported mlx5 classes supplied.");
305 /* Default to net class. */
306 user_classes = MLX5_CLASS_NET;
308 dev = pci_to_mlx5_device(pci_dev);
310 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0);
313 dev->pci_dev = pci_dev;
314 TAILQ_INSERT_HEAD(&devices_list, dev, next);
317 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes);
328 * DPDK callback to remove one or more drivers for a PCI device.
330 * This function removes all drivers probed for a given PCI device.
333 * Pointer to the PCI device.
336 * 0 on success, the function cannot fail.
339 mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
341 struct mlx5_pci_device *dev;
344 dev = pci_to_mlx5_device(pci_dev);
347 /* Matching device found, cleanup and unload drivers. */
348 ret = drivers_remove(dev, dev->classes_loaded);
355 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
356 uint64_t iova, size_t len)
358 struct mlx5_pci_driver *driver = NULL;
359 struct mlx5_pci_driver *temp;
360 struct mlx5_pci_device *dev;
363 dev = pci_to_mlx5_device(pci_dev);
366 TAILQ_FOREACH(driver, &drv_list, next) {
367 if (device_class_enabled(dev, driver->driver_class) &&
368 driver->pci_driver.dma_map) {
369 ret = driver->pci_driver.dma_map(pci_dev, addr,
377 TAILQ_FOREACH(temp, &drv_list, next) {
380 if (device_class_enabled(dev, temp->driver_class) &&
381 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap)
382 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len);
388 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
389 uint64_t iova, size_t len)
391 struct mlx5_pci_driver *driver;
392 struct mlx5_pci_device *dev;
393 int local_ret = -EINVAL;
396 dev = pci_to_mlx5_device(pci_dev);
400 /* There is no unmap error recovery in current implementation. */
401 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) {
402 if (device_class_enabled(dev, driver->driver_class) &&
403 driver->pci_driver.dma_unmap) {
404 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr,
406 if (local_ret && (ret == 0))
415 /* PCI ID table is build dynamically based on registered mlx5 drivers. */
416 static struct rte_pci_id *mlx5_pci_id_table;
418 static struct rte_pci_driver mlx5_pci_driver = {
420 .name = MLX5_PCI_DRIVER_NAME,
422 .probe = mlx5_common_pci_probe,
423 .remove = mlx5_common_pci_remove,
424 .dma_map = mlx5_common_pci_dma_map,
425 .dma_unmap = mlx5_common_pci_dma_unmap,
429 pci_id_table_size_get(const struct rte_pci_id *id_table)
433 for (; id_table->vendor_id != 0; id_table++)
439 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table,
442 int current_size = next_idx - 1;
445 for (i = 0; i < current_size; i++) {
446 if (id->device_id == table[i].device_id &&
447 id->vendor_id == table[i].vendor_id &&
448 id->subsystem_vendor_id == table[i].subsystem_vendor_id &&
449 id->subsystem_device_id == table[i].subsystem_device_id)
456 pci_id_insert(struct rte_pci_id *new_table, int *next_idx,
457 const struct rte_pci_id *id_table)
459 /* Traverse the id_table, check if entry exists in new_table;
460 * Add non duplicate entries to new table.
462 for (; id_table->vendor_id != 0; id_table++) {
463 if (!pci_id_exists(id_table, new_table, *next_idx)) {
464 /* New entry; add to the table. */
465 new_table[*next_idx] = *id_table;
472 pci_ids_table_update(const struct rte_pci_id *driver_id_table)
474 const struct rte_pci_id *id_iter;
475 struct rte_pci_id *updated_table;
476 struct rte_pci_id *old_table;
480 old_table = mlx5_pci_id_table;
482 num_ids = pci_id_table_size_get(old_table);
483 num_ids += pci_id_table_size_get(driver_id_table);
484 /* Increase size by one for the termination entry of vendor_id = 0. */
486 updated_table = calloc(num_ids, sizeof(*updated_table));
489 if (TAILQ_EMPTY(&drv_list)) {
490 /* Copy the first driver's ID table. */
491 for (id_iter = driver_id_table; id_iter->vendor_id != 0;
493 updated_table[i] = *id_iter;
495 /* First copy existing table entries. */
496 for (id_iter = old_table; id_iter->vendor_id != 0;
498 updated_table[i] = *id_iter;
499 /* New id to be added at the end of current ID table. */
500 pci_id_insert(updated_table, &i, driver_id_table);
502 /* Terminate table with empty entry. */
503 updated_table[i].vendor_id = 0;
504 mlx5_pci_driver.id_table = updated_table;
505 mlx5_pci_id_table = updated_table;
512 mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
516 ret = pci_ids_table_update(driver->pci_driver.id_table);
519 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags;
520 TAILQ_INSERT_TAIL(&drv_list, driver, next);
523 void mlx5_common_pci_init(void)
525 const struct rte_pci_id empty_table[] = {
531 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD
532 * including this one can register the PCI table first. If any other
533 * PMD(s) have registered the PCI ID table, No need to register an empty
536 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table))
538 rte_pci_register(&mlx5_pci_driver);
541 RTE_FINI(mlx5_common_pci_finish)
543 if (mlx5_pci_id_table != NULL) {
544 /* Constructor doesn't register with PCI bus if it failed
545 * to build the table.
547 rte_pci_unregister(&mlx5_pci_driver);
548 free(mlx5_pci_id_table);
551 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__);