1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies Ltd
6 #include <rte_malloc.h>
7 #include "mlx5_common_utils.h"
8 #include "mlx5_common_pci.h"
10 struct mlx5_pci_device {
11 struct rte_pci_device *pci_dev;
12 TAILQ_ENTRY(mlx5_pci_device) next;
13 uint32_t classes_loaded;
16 /* Head of list of drivers. */
17 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list =
18 TAILQ_HEAD_INITIALIZER(drv_list);
20 /* Head of mlx5 pci devices. */
21 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
22 TAILQ_HEAD_INITIALIZER(devices_list);
26 unsigned int driver_class;
28 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA },
29 { .name = "net", .driver_class = MLX5_CLASS_NET },
30 { .name = "regex", .driver_class = MLX5_CLASS_REGEX },
31 { .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
34 static const unsigned int mlx5_class_combinations[] = {
39 MLX5_CLASS_NET | MLX5_CLASS_REGEX,
40 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
41 MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
42 MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
43 MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
44 MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
45 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
46 /* New class combination should be added here. */
50 class_name_to_value(const char *class_name)
54 for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
55 if (strcmp(class_name, mlx5_classes[i].name) == 0)
56 return mlx5_classes[i].driver_class;
61 static struct mlx5_pci_driver *
62 driver_get(uint32_t class)
64 struct mlx5_pci_driver *driver;
66 TAILQ_FOREACH(driver, &drv_list, next) {
67 if (driver->driver_class == class)
74 bus_cmdline_options_handler(__rte_unused const char *key,
75 const char *class_names, void *opaque)
85 nstr = strdup(class_names);
91 found = strtok_r(nstr, ":", &refstr);
95 /* Extract each individual class name. Multiple
96 * class key,value is supplied as class=net:vdpa:foo:bar.
98 class_val = class_name_to_value(found);
99 /* Check if its a valid class. */
105 found = strtok_r(NULL, ":", &refstr);
110 DRV_LOG(ERR, "Invalid mlx5 class options %s."
111 " Maybe typo in device class argument setting?",
117 parse_class_options(const struct rte_devargs *devargs)
119 const char *key = MLX5_CLASS_ARG_NAME;
120 struct rte_kvargs *kvlist;
125 kvlist = rte_kvargs_parse(devargs->args, NULL);
128 if (rte_kvargs_count(kvlist, key))
129 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler,
131 rte_kvargs_free(kvlist);
136 mlx5_bus_match(const struct mlx5_pci_driver *drv,
137 const struct rte_pci_device *pci_dev)
139 const struct rte_pci_id *id_table;
141 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0;
143 /* Check if device's ids match the class driver's ids. */
144 if (id_table->vendor_id != pci_dev->id.vendor_id &&
145 id_table->vendor_id != PCI_ANY_ID)
147 if (id_table->device_id != pci_dev->id.device_id &&
148 id_table->device_id != PCI_ANY_ID)
150 if (id_table->subsystem_vendor_id !=
151 pci_dev->id.subsystem_vendor_id &&
152 id_table->subsystem_vendor_id != PCI_ANY_ID)
154 if (id_table->subsystem_device_id !=
155 pci_dev->id.subsystem_device_id &&
156 id_table->subsystem_device_id != PCI_ANY_ID)
158 if (id_table->class_id != pci_dev->id.class_id &&
159 id_table->class_id != RTE_CLASS_ANY_ID)
167 is_valid_class_combination(uint32_t user_classes)
171 /* Verify if user specified valid supported combination. */
172 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) {
173 if (mlx5_class_combinations[i] == user_classes)
176 /* Not found any valid class combination. */
180 static struct mlx5_pci_device *
181 pci_to_mlx5_device(const struct rte_pci_device *pci_dev)
183 struct mlx5_pci_device *dev;
185 TAILQ_FOREACH(dev, &devices_list, next) {
186 if (dev->pci_dev == pci_dev)
193 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class)
195 return (device->classes_loaded & class) ? true : false;
199 dev_release(struct mlx5_pci_device *dev)
201 TAILQ_REMOVE(&devices_list, dev, next);
206 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes)
208 struct mlx5_pci_driver *driver;
209 int local_ret = -ENODEV;
213 enabled_classes &= dev->classes_loaded;
214 while (enabled_classes) {
215 driver = driver_get(RTE_BIT64(i));
217 local_ret = driver->pci_driver.remove(dev->pci_dev);
219 dev->classes_loaded &= ~RTE_BIT64(i);
223 enabled_classes &= ~RTE_BIT64(i);
232 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv,
233 struct rte_pci_device *pci_dev, uint32_t user_classes)
235 struct mlx5_pci_driver *driver;
236 uint32_t enabled_classes = 0;
240 TAILQ_FOREACH(driver, &drv_list, next) {
241 if ((driver->driver_class & user_classes) == 0)
243 if (!mlx5_bus_match(driver, pci_dev))
245 already_loaded = dev->classes_loaded & driver->driver_class;
246 if (already_loaded &&
247 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
248 DRV_LOG(ERR, "Device %s is already probed\n",
249 pci_dev->device.name);
253 ret = driver->pci_driver.probe(pci_drv, pci_dev);
255 DRV_LOG(ERR, "Failed to load driver = %s.\n",
256 driver->pci_driver.driver.name);
259 enabled_classes |= driver->driver_class;
261 dev->classes_loaded |= enabled_classes;
264 /* Only unload drivers which are enabled which were enabled
265 * in this probe instance.
267 drivers_remove(dev, enabled_classes);
272 * DPDK callback to register to probe multiple drivers for a PCI device.
275 * PCI driver structure.
277 * PCI device information.
280 * 0 on success, a negative errno value otherwise and rte_errno is set.
283 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
284 struct rte_pci_device *pci_dev)
286 struct mlx5_pci_device *dev;
287 uint32_t user_classes = 0;
288 bool new_device = false;
291 ret = parse_class_options(pci_dev->device.devargs);
296 /* Validate combination here. */
297 ret = is_valid_class_combination(user_classes);
299 DRV_LOG(ERR, "Unsupported mlx5 classes supplied.");
303 /* Default to net class. */
304 user_classes = MLX5_CLASS_NET;
306 dev = pci_to_mlx5_device(pci_dev);
308 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0);
311 dev->pci_dev = pci_dev;
312 TAILQ_INSERT_HEAD(&devices_list, dev, next);
315 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes);
326 * DPDK callback to remove one or more drivers for a PCI device.
328 * This function removes all drivers probed for a given PCI device.
331 * Pointer to the PCI device.
334 * 0 on success, the function cannot fail.
337 mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
339 struct mlx5_pci_device *dev;
342 dev = pci_to_mlx5_device(pci_dev);
345 /* Matching device found, cleanup and unload drivers. */
346 ret = drivers_remove(dev, dev->classes_loaded);
353 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
354 uint64_t iova, size_t len)
356 struct mlx5_pci_driver *driver = NULL;
357 struct mlx5_pci_driver *temp;
358 struct mlx5_pci_device *dev;
361 dev = pci_to_mlx5_device(pci_dev);
364 TAILQ_FOREACH(driver, &drv_list, next) {
365 if (device_class_enabled(dev, driver->driver_class) &&
366 driver->pci_driver.dma_map) {
367 ret = driver->pci_driver.dma_map(pci_dev, addr,
375 TAILQ_FOREACH(temp, &drv_list, next) {
378 if (device_class_enabled(dev, temp->driver_class) &&
379 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap)
380 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len);
386 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
387 uint64_t iova, size_t len)
389 struct mlx5_pci_driver *driver;
390 struct mlx5_pci_device *dev;
391 int local_ret = -EINVAL;
394 dev = pci_to_mlx5_device(pci_dev);
398 /* There is no unmap error recovery in current implementation. */
399 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) {
400 if (device_class_enabled(dev, driver->driver_class) &&
401 driver->pci_driver.dma_unmap) {
402 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr,
404 if (local_ret && (ret == 0))
413 /* PCI ID table is build dynamically based on registered mlx5 drivers. */
414 static struct rte_pci_id *mlx5_pci_id_table;
416 static struct rte_pci_driver mlx5_pci_driver = {
418 .name = MLX5_DRIVER_NAME,
420 .probe = mlx5_common_pci_probe,
421 .remove = mlx5_common_pci_remove,
422 .dma_map = mlx5_common_pci_dma_map,
423 .dma_unmap = mlx5_common_pci_dma_unmap,
427 pci_id_table_size_get(const struct rte_pci_id *id_table)
431 for (; id_table->vendor_id != 0; id_table++)
437 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table,
440 int current_size = next_idx - 1;
443 for (i = 0; i < current_size; i++) {
444 if (id->device_id == table[i].device_id &&
445 id->vendor_id == table[i].vendor_id &&
446 id->subsystem_vendor_id == table[i].subsystem_vendor_id &&
447 id->subsystem_device_id == table[i].subsystem_device_id)
454 pci_id_insert(struct rte_pci_id *new_table, int *next_idx,
455 const struct rte_pci_id *id_table)
457 /* Traverse the id_table, check if entry exists in new_table;
458 * Add non duplicate entries to new table.
460 for (; id_table->vendor_id != 0; id_table++) {
461 if (!pci_id_exists(id_table, new_table, *next_idx)) {
462 /* New entry; add to the table. */
463 new_table[*next_idx] = *id_table;
470 pci_ids_table_update(const struct rte_pci_id *driver_id_table)
472 const struct rte_pci_id *id_iter;
473 struct rte_pci_id *updated_table;
474 struct rte_pci_id *old_table;
478 old_table = mlx5_pci_id_table;
480 num_ids = pci_id_table_size_get(old_table);
481 num_ids += pci_id_table_size_get(driver_id_table);
482 /* Increase size by one for the termination entry of vendor_id = 0. */
484 updated_table = calloc(num_ids, sizeof(*updated_table));
487 if (TAILQ_EMPTY(&drv_list)) {
488 /* Copy the first driver's ID table. */
489 for (id_iter = driver_id_table; id_iter->vendor_id != 0;
491 updated_table[i] = *id_iter;
493 /* First copy existing table entries. */
494 for (id_iter = old_table; id_iter->vendor_id != 0;
496 updated_table[i] = *id_iter;
497 /* New id to be added at the end of current ID table. */
498 pci_id_insert(updated_table, &i, driver_id_table);
500 /* Terminate table with empty entry. */
501 updated_table[i].vendor_id = 0;
502 mlx5_pci_driver.id_table = updated_table;
503 mlx5_pci_id_table = updated_table;
510 mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
514 ret = pci_ids_table_update(driver->pci_driver.id_table);
517 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags;
518 TAILQ_INSERT_TAIL(&drv_list, driver, next);
521 void mlx5_common_pci_init(void)
523 const struct rte_pci_id empty_table[] = {
529 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD
530 * including this one can register the PCI table first. If any other
531 * PMD(s) have registered the PCI ID table, No need to register an empty
534 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table))
536 rte_pci_register(&mlx5_pci_driver);
539 RTE_FINI(mlx5_common_pci_finish)
541 if (mlx5_pci_id_table != NULL) {
542 /* Constructor doesn't register with PCI bus if it failed
543 * to build the table.
545 rte_pci_unregister(&mlx5_pci_driver);
546 free(mlx5_pci_id_table);
549 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__);