1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies Ltd
6 #include <rte_malloc.h>
7 #include "mlx5_common_utils.h"
8 #include "mlx5_common_pci.h"
10 struct mlx5_pci_device {
11 struct rte_pci_device *pci_dev;
12 TAILQ_ENTRY(mlx5_pci_device) next;
13 uint32_t classes_loaded;
16 /* Head of list of drivers. */
17 static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list =
18 TAILQ_HEAD_INITIALIZER(drv_list);
20 /* Head of mlx5 pci devices. */
21 static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
22 TAILQ_HEAD_INITIALIZER(devices_list);
26 unsigned int driver_class;
28 { .name = "vdpa", .driver_class = MLX5_CLASS_VDPA },
29 { .name = "net", .driver_class = MLX5_CLASS_NET },
30 { .name = "regex", .driver_class = MLX5_CLASS_REGEX },
33 static const unsigned int mlx5_class_combinations[] = {
37 MLX5_CLASS_NET | MLX5_CLASS_REGEX,
38 MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
39 /* New class combination should be added here. */
43 class_name_to_value(const char *class_name)
47 for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
48 if (strcmp(class_name, mlx5_classes[i].name) == 0)
49 return mlx5_classes[i].driver_class;
54 static struct mlx5_pci_driver *
55 driver_get(uint32_t class)
57 struct mlx5_pci_driver *driver;
59 TAILQ_FOREACH(driver, &drv_list, next) {
60 if (driver->driver_class == class)
67 bus_cmdline_options_handler(__rte_unused const char *key,
68 const char *class_names, void *opaque)
78 nstr = strdup(class_names);
84 found = strtok_r(nstr, ":", &refstr);
88 /* Extract each individual class name. Multiple
89 * class key,value is supplied as class=net:vdpa:foo:bar.
91 class_val = class_name_to_value(found);
92 /* Check if its a valid class. */
98 found = strtok_r(NULL, ":", &refstr);
103 DRV_LOG(ERR, "Invalid mlx5 class options %s."
104 " Maybe typo in device class argument setting?",
110 parse_class_options(const struct rte_devargs *devargs)
112 const char *key = MLX5_CLASS_ARG_NAME;
113 struct rte_kvargs *kvlist;
118 kvlist = rte_kvargs_parse(devargs->args, NULL);
121 if (rte_kvargs_count(kvlist, key))
122 rte_kvargs_process(kvlist, key, bus_cmdline_options_handler,
124 rte_kvargs_free(kvlist);
129 mlx5_bus_match(const struct mlx5_pci_driver *drv,
130 const struct rte_pci_device *pci_dev)
132 const struct rte_pci_id *id_table;
134 for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0;
136 /* Check if device's ids match the class driver's ids. */
137 if (id_table->vendor_id != pci_dev->id.vendor_id &&
138 id_table->vendor_id != PCI_ANY_ID)
140 if (id_table->device_id != pci_dev->id.device_id &&
141 id_table->device_id != PCI_ANY_ID)
143 if (id_table->subsystem_vendor_id !=
144 pci_dev->id.subsystem_vendor_id &&
145 id_table->subsystem_vendor_id != PCI_ANY_ID)
147 if (id_table->subsystem_device_id !=
148 pci_dev->id.subsystem_device_id &&
149 id_table->subsystem_device_id != PCI_ANY_ID)
151 if (id_table->class_id != pci_dev->id.class_id &&
152 id_table->class_id != RTE_CLASS_ANY_ID)
160 is_valid_class_combination(uint32_t user_classes)
164 /* Verify if user specified valid supported combination. */
165 for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) {
166 if (mlx5_class_combinations[i] == user_classes)
169 /* Not found any valid class combination. */
173 static struct mlx5_pci_device *
174 pci_to_mlx5_device(const struct rte_pci_device *pci_dev)
176 struct mlx5_pci_device *dev;
178 TAILQ_FOREACH(dev, &devices_list, next) {
179 if (dev->pci_dev == pci_dev)
186 device_class_enabled(const struct mlx5_pci_device *device, uint32_t class)
188 return (device->classes_loaded & class) ? true : false;
192 dev_release(struct mlx5_pci_device *dev)
194 TAILQ_REMOVE(&devices_list, dev, next);
199 drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes)
201 struct mlx5_pci_driver *driver;
202 int local_ret = -ENODEV;
206 enabled_classes &= dev->classes_loaded;
207 while (enabled_classes) {
208 driver = driver_get(RTE_BIT64(i));
210 local_ret = driver->pci_driver.remove(dev->pci_dev);
212 dev->classes_loaded &= ~RTE_BIT64(i);
216 enabled_classes &= ~RTE_BIT64(i);
225 drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv,
226 struct rte_pci_device *pci_dev, uint32_t user_classes)
228 struct mlx5_pci_driver *driver;
229 uint32_t enabled_classes = 0;
233 TAILQ_FOREACH(driver, &drv_list, next) {
234 if ((driver->driver_class & user_classes) == 0)
236 if (!mlx5_bus_match(driver, pci_dev))
238 already_loaded = dev->classes_loaded & driver->driver_class;
239 if (already_loaded &&
240 !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
241 DRV_LOG(ERR, "Device %s is already probed\n",
242 pci_dev->device.name);
246 ret = driver->pci_driver.probe(pci_drv, pci_dev);
248 DRV_LOG(ERR, "Failed to load driver = %s.\n",
249 driver->pci_driver.driver.name);
252 enabled_classes |= driver->driver_class;
254 dev->classes_loaded |= enabled_classes;
257 /* Only unload drivers which are enabled which were enabled
258 * in this probe instance.
260 drivers_remove(dev, enabled_classes);
265 * DPDK callback to register to probe multiple drivers for a PCI device.
268 * PCI driver structure.
270 * PCI device information.
273 * 0 on success, a negative errno value otherwise and rte_errno is set.
276 mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
277 struct rte_pci_device *pci_dev)
279 struct mlx5_pci_device *dev;
280 uint32_t user_classes = 0;
281 bool new_device = false;
284 ret = parse_class_options(pci_dev->device.devargs);
289 /* Validate combination here. */
290 ret = is_valid_class_combination(user_classes);
292 DRV_LOG(ERR, "Unsupported mlx5 classes supplied.");
296 /* Default to net class. */
297 user_classes = MLX5_CLASS_NET;
299 dev = pci_to_mlx5_device(pci_dev);
301 dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0);
304 dev->pci_dev = pci_dev;
305 TAILQ_INSERT_HEAD(&devices_list, dev, next);
308 ret = drivers_probe(dev, pci_drv, pci_dev, user_classes);
319 * DPDK callback to remove one or more drivers for a PCI device.
321 * This function removes all drivers probed for a given PCI device.
324 * Pointer to the PCI device.
327 * 0 on success, the function cannot fail.
330 mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
332 struct mlx5_pci_device *dev;
335 dev = pci_to_mlx5_device(pci_dev);
338 /* Matching device found, cleanup and unload drivers. */
339 ret = drivers_remove(dev, dev->classes_loaded);
346 mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
347 uint64_t iova, size_t len)
349 struct mlx5_pci_driver *driver = NULL;
350 struct mlx5_pci_driver *temp;
351 struct mlx5_pci_device *dev;
354 dev = pci_to_mlx5_device(pci_dev);
357 TAILQ_FOREACH(driver, &drv_list, next) {
358 if (device_class_enabled(dev, driver->driver_class) &&
359 driver->pci_driver.dma_map) {
360 ret = driver->pci_driver.dma_map(pci_dev, addr,
368 TAILQ_FOREACH(temp, &drv_list, next) {
371 if (device_class_enabled(dev, temp->driver_class) &&
372 temp->pci_driver.dma_map && temp->pci_driver.dma_unmap)
373 temp->pci_driver.dma_unmap(pci_dev, addr, iova, len);
379 mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
380 uint64_t iova, size_t len)
382 struct mlx5_pci_driver *driver;
383 struct mlx5_pci_device *dev;
384 int local_ret = -EINVAL;
387 dev = pci_to_mlx5_device(pci_dev);
391 /* There is no unmap error recovery in current implementation. */
392 TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) {
393 if (device_class_enabled(dev, driver->driver_class) &&
394 driver->pci_driver.dma_unmap) {
395 local_ret = driver->pci_driver.dma_unmap(pci_dev, addr,
397 if (local_ret && (ret == 0))
406 /* PCI ID table is build dynamically based on registered mlx5 drivers. */
407 static struct rte_pci_id *mlx5_pci_id_table;
409 static struct rte_pci_driver mlx5_pci_driver = {
413 .probe = mlx5_common_pci_probe,
414 .remove = mlx5_common_pci_remove,
415 .dma_map = mlx5_common_pci_dma_map,
416 .dma_unmap = mlx5_common_pci_dma_unmap,
420 pci_id_table_size_get(const struct rte_pci_id *id_table)
424 for (; id_table->vendor_id != 0; id_table++)
430 pci_id_exists(const struct rte_pci_id *id, const struct rte_pci_id *table,
433 int current_size = next_idx - 1;
436 for (i = 0; i < current_size; i++) {
437 if (id->device_id == table[i].device_id &&
438 id->vendor_id == table[i].vendor_id &&
439 id->subsystem_vendor_id == table[i].subsystem_vendor_id &&
440 id->subsystem_device_id == table[i].subsystem_device_id)
447 pci_id_insert(struct rte_pci_id *new_table, int *next_idx,
448 const struct rte_pci_id *id_table)
450 /* Traverse the id_table, check if entry exists in new_table;
451 * Add non duplicate entries to new table.
453 for (; id_table->vendor_id != 0; id_table++) {
454 if (!pci_id_exists(id_table, new_table, *next_idx)) {
455 /* New entry; add to the table. */
456 new_table[*next_idx] = *id_table;
463 pci_ids_table_update(const struct rte_pci_id *driver_id_table)
465 const struct rte_pci_id *id_iter;
466 struct rte_pci_id *updated_table;
467 struct rte_pci_id *old_table;
471 old_table = mlx5_pci_id_table;
473 num_ids = pci_id_table_size_get(old_table);
474 num_ids += pci_id_table_size_get(driver_id_table);
475 /* Increase size by one for the termination entry of vendor_id = 0. */
477 updated_table = calloc(num_ids, sizeof(*updated_table));
480 if (TAILQ_EMPTY(&drv_list)) {
481 /* Copy the first driver's ID table. */
482 for (id_iter = driver_id_table; id_iter->vendor_id != 0;
484 updated_table[i] = *id_iter;
486 /* First copy existing table entries. */
487 for (id_iter = old_table; id_iter->vendor_id != 0;
489 updated_table[i] = *id_iter;
490 /* New id to be added at the end of current ID table. */
491 pci_id_insert(updated_table, &i, driver_id_table);
493 /* Terminate table with empty entry. */
494 updated_table[i].vendor_id = 0;
495 mlx5_pci_driver.id_table = updated_table;
496 mlx5_pci_id_table = updated_table;
503 mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
507 ret = pci_ids_table_update(driver->pci_driver.id_table);
510 mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags;
511 TAILQ_INSERT_TAIL(&drv_list, driver, next);
514 void mlx5_common_pci_init(void)
516 const struct rte_pci_id empty_table[] = {
522 /* All mlx5 PMDs constructor runs at same priority. So any of the PMD
523 * including this one can register the PCI table first. If any other
524 * PMD(s) have registered the PCI ID table, No need to register an empty
527 if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table))
529 rte_pci_register(&mlx5_pci_driver);
532 RTE_FINI(mlx5_common_pci_finish)
534 if (mlx5_pci_id_table != NULL) {
535 /* Constructor doesn't register with PCI bus if it failed
536 * to build the table.
538 rte_pci_unregister(&mlx5_pci_driver);
539 free(mlx5_pci_id_table);
542 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__);