1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 Mellanox Technologies, Ltd
10 #include <rte_mempool.h>
11 #include <rte_class.h>
12 #include <rte_malloc.h>
14 #include "mlx5_common.h"
15 #include "mlx5_common_os.h"
16 #include "mlx5_common_log.h"
17 #include "mlx5_common_pci.h"
18 #include "mlx5_common_private.h"
20 uint8_t haswell_broadwell_cpu;
22 /* In case this is an x86_64 intel processor to check if
23 * we should use relaxed ordering.
25 #ifdef RTE_ARCH_X86_64
27 * This function returns processor identification and feature information
30 * @param eax, ebx, ecx, edx
31 * Pointers to the registers that will hold cpu information.
33 * The main category of information returned.
35 static inline void mlx5_cpu_id(unsigned int level,
36 unsigned int *eax, unsigned int *ebx,
37 unsigned int *ecx, unsigned int *edx)
40 : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx)
45 RTE_LOG_REGISTER_DEFAULT(mlx5_common_logtype, NOTICE)
47 /* Head of list of drivers. */
48 static TAILQ_HEAD(mlx5_drivers, mlx5_class_driver) drivers_list =
49 TAILQ_HEAD_INITIALIZER(drivers_list);
51 /* Head of devices. */
52 static TAILQ_HEAD(mlx5_devices, mlx5_common_device) devices_list =
53 TAILQ_HEAD_INITIALIZER(devices_list);
57 unsigned int drv_class;
59 { .name = "vdpa", .drv_class = MLX5_CLASS_VDPA },
60 { .name = "eth", .drv_class = MLX5_CLASS_ETH },
61 /* Keep class "net" for backward compatibility. */
62 { .name = "net", .drv_class = MLX5_CLASS_ETH },
63 { .name = "regex", .drv_class = MLX5_CLASS_REGEX },
64 { .name = "compress", .drv_class = MLX5_CLASS_COMPRESS },
65 { .name = "crypto", .drv_class = MLX5_CLASS_CRYPTO },
69 class_name_to_value(const char *class_name)
73 for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
74 if (strcmp(class_name, mlx5_classes[i].name) == 0)
75 return mlx5_classes[i].drv_class;
80 static struct mlx5_class_driver *
81 driver_get(uint32_t class)
83 struct mlx5_class_driver *driver;
85 TAILQ_FOREACH(driver, &drivers_list, next) {
86 if ((uint32_t)driver->drv_class == class)
93 devargs_class_handler(__rte_unused const char *key,
94 const char *class_names, void *opaque)
103 scratch = strdup(class_names);
104 if (scratch == NULL) {
108 found = strtok_r(scratch, ":", &refstr);
113 /* Extract each individual class name. Multiple
114 * classes can be supplied as class=net:regex:foo:bar.
116 class_val = class_name_to_value(found);
117 /* Check if its a valid class. */
123 found = strtok_r(NULL, ":", &refstr);
124 } while (found != NULL);
128 DRV_LOG(ERR, "Invalid mlx5 class options: %s.\n", class_names);
133 parse_class_options(const struct rte_devargs *devargs)
135 struct rte_kvargs *kvlist;
140 if (devargs->cls != NULL && devargs->cls->name != NULL)
141 /* Global syntax, only one class type. */
142 return class_name_to_value(devargs->cls->name);
143 /* Legacy devargs support multiple classes. */
144 kvlist = rte_kvargs_parse(devargs->args, NULL);
147 rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_CLASS,
148 devargs_class_handler, &ret);
149 rte_kvargs_free(kvlist);
153 static const unsigned int mlx5_class_invalid_combinations[] = {
154 MLX5_CLASS_ETH | MLX5_CLASS_VDPA,
155 /* New class combination should be added here. */
159 is_valid_class_combination(uint32_t user_classes)
163 /* Verify if user specified unsupported combination. */
164 for (i = 0; i < RTE_DIM(mlx5_class_invalid_combinations); i++) {
165 if ((mlx5_class_invalid_combinations[i] & user_classes) ==
166 mlx5_class_invalid_combinations[i])
169 /* Not found any invalid class combination. */
174 device_class_enabled(const struct mlx5_common_device *device, uint32_t class)
176 return (device->classes_loaded & class) > 0;
180 mlx5_bus_match(const struct mlx5_class_driver *drv,
181 const struct rte_device *dev)
183 if (mlx5_dev_is_pci(dev))
184 return mlx5_dev_pci_match(drv, dev);
188 static struct mlx5_common_device *
189 to_mlx5_device(const struct rte_device *rte_dev)
191 struct mlx5_common_device *dev;
193 TAILQ_FOREACH(dev, &devices_list, next) {
194 if (rte_dev == dev->dev)
201 dev_release(struct mlx5_common_device *dev)
203 TAILQ_REMOVE(&devices_list, dev, next);
208 drivers_remove(struct mlx5_common_device *dev, uint32_t enabled_classes)
210 struct mlx5_class_driver *driver;
211 int local_ret = -ENODEV;
215 enabled_classes &= dev->classes_loaded;
216 while (enabled_classes) {
217 driver = driver_get(RTE_BIT64(i));
218 if (driver != NULL) {
219 local_ret = driver->remove(dev->dev);
221 dev->classes_loaded &= ~RTE_BIT64(i);
225 enabled_classes &= ~RTE_BIT64(i);
228 if (local_ret != 0 && ret == 0)
234 drivers_probe(struct mlx5_common_device *dev, uint32_t user_classes)
236 struct mlx5_class_driver *driver;
237 uint32_t enabled_classes = 0;
241 TAILQ_FOREACH(driver, &drivers_list, next) {
242 if ((driver->drv_class & user_classes) == 0)
244 if (!mlx5_bus_match(driver, dev->dev))
246 already_loaded = dev->classes_loaded & driver->drv_class;
247 if (already_loaded && driver->probe_again == 0) {
248 DRV_LOG(ERR, "Device %s is already probed",
253 ret = driver->probe(dev->dev);
255 DRV_LOG(ERR, "Failed to load driver %s",
259 enabled_classes |= driver->drv_class;
261 dev->classes_loaded |= enabled_classes;
264 /* Only unload drivers which are enabled which were enabled
265 * in this probe instance.
267 drivers_remove(dev, enabled_classes);
272 mlx5_common_dev_probe(struct rte_device *eal_dev)
274 struct mlx5_common_device *dev;
275 uint32_t classes = 0;
276 bool new_device = false;
279 DRV_LOG(INFO, "probe device \"%s\".", eal_dev->name);
280 ret = parse_class_options(eal_dev->devargs);
282 DRV_LOG(ERR, "Unsupported mlx5 class type: %s",
283 eal_dev->devargs->args);
288 /* Default to net class. */
289 classes = MLX5_CLASS_ETH;
290 dev = to_mlx5_device(eal_dev);
292 dev = rte_zmalloc("mlx5_common_device", sizeof(*dev), 0);
296 TAILQ_INSERT_HEAD(&devices_list, dev, next);
299 /* Validate combination here. */
300 ret = is_valid_class_combination(classes |
301 dev->classes_loaded);
303 DRV_LOG(ERR, "Unsupported mlx5 classes combination.");
307 ret = drivers_probe(dev, classes);
318 mlx5_common_dev_remove(struct rte_device *eal_dev)
320 struct mlx5_common_device *dev;
323 dev = to_mlx5_device(eal_dev);
326 /* Matching device found, cleanup and unload drivers. */
327 ret = drivers_remove(dev, dev->classes_loaded);
334 mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova,
337 struct mlx5_class_driver *driver = NULL;
338 struct mlx5_class_driver *temp;
339 struct mlx5_common_device *mdev;
342 mdev = to_mlx5_device(dev);
345 TAILQ_FOREACH(driver, &drivers_list, next) {
346 if (!device_class_enabled(mdev, driver->drv_class) ||
347 driver->dma_map == NULL)
349 ret = driver->dma_map(dev, addr, iova, len);
355 TAILQ_FOREACH(temp, &drivers_list, next) {
358 if (device_class_enabled(mdev, temp->drv_class) &&
359 temp->dma_map && temp->dma_unmap)
360 temp->dma_unmap(dev, addr, iova, len);
366 mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
369 struct mlx5_class_driver *driver;
370 struct mlx5_common_device *mdev;
371 int local_ret = -EINVAL;
374 mdev = to_mlx5_device(dev);
377 /* There is no unmap error recovery in current implementation. */
378 TAILQ_FOREACH_REVERSE(driver, &drivers_list, mlx5_drivers, next) {
379 if (!device_class_enabled(mdev, driver->drv_class) ||
380 driver->dma_unmap == NULL)
382 local_ret = driver->dma_unmap(dev, addr, iova, len);
383 if (local_ret && (ret == 0))
392 mlx5_class_driver_register(struct mlx5_class_driver *driver)
394 mlx5_common_driver_on_register_pci(driver);
395 TAILQ_INSERT_TAIL(&drivers_list, driver, next);
398 static void mlx5_common_driver_init(void)
400 mlx5_common_pci_init();
403 static bool mlx5_common_initialized;
406 * One time innitialization routine for run-time dependency on glue library
407 * for multiple PMDs. Each mlx5 PMD that depends on mlx5_common module,
408 * must invoke in its constructor.
411 mlx5_common_init(void)
413 if (mlx5_common_initialized)
416 mlx5_glue_constructor();
417 mlx5_common_driver_init();
418 mlx5_common_initialized = true;
422 * This function is responsible of initializing the variable
423 * haswell_broadwell_cpu by checking if the cpu is intel
424 * and reading the data returned from mlx5_cpu_id().
425 * since haswell and broadwell cpus don't have improved performance
426 * when using relaxed ordering we want to check the cpu type before
427 * before deciding whether to enable RO or not.
428 * if the cpu is haswell or broadwell the variable will be set to 1
429 * otherwise it will be 0.
431 RTE_INIT_PRIO(mlx5_is_haswell_broadwell_cpu, LOG)
433 #ifdef RTE_ARCH_X86_64
434 unsigned int broadwell_models[4] = {0x3d, 0x47, 0x4F, 0x56};
435 unsigned int haswell_models[4] = {0x3c, 0x3f, 0x45, 0x46};
436 unsigned int i, model, family, brand_id, vendor;
437 unsigned int signature_intel_ebx = 0x756e6547;
438 unsigned int extended_model;
439 unsigned int eax = 0;
440 unsigned int ebx = 0;
441 unsigned int ecx = 0;
442 unsigned int edx = 0;
445 mlx5_cpu_id(0, &eax, &ebx, &ecx, &edx);
449 haswell_broadwell_cpu = 0;
452 mlx5_cpu_id(1, &eax, &ebx, &ecx, &edx);
453 model = (eax >> 4) & 0x0f;
454 family = (eax >> 8) & 0x0f;
455 brand_id = ebx & 0xff;
456 extended_model = (eax >> 12) & 0xf0;
457 /* Check if the processor is Haswell or Broadwell */
458 if (vendor == signature_intel_ebx) {
460 model += extended_model;
461 if (brand_id == 0 && family == 0x6) {
462 for (i = 0; i < RTE_DIM(broadwell_models); i++)
463 if (model == broadwell_models[i]) {
464 haswell_broadwell_cpu = 1;
467 for (i = 0; i < RTE_DIM(haswell_models); i++)
468 if (model == haswell_models[i]) {
469 haswell_broadwell_cpu = 1;
475 haswell_broadwell_cpu = 0;
479 * Allocate the User Access Region with DevX on specified device.
482 * Infiniband device context to perform allocation on.
483 * @param [in] mapping
484 * MLX5DV_UAR_ALLOC_TYPE_BF - allocate as cached memory with write-combining
485 * attributes (if supported by the host), the
486 * writes to the UAR registers must be followed
487 * by write memory barrier.
488 * MLX5DV_UAR_ALLOC_TYPE_NC - allocate as non-cached nenory, all writes are
489 * promoted to the registers immediately, no
490 * memory barriers needed.
491 * mapping < 0 - the first attempt is performed with MLX5DV_UAR_ALLOC_TYPE_BF,
492 * if this fails the next attempt with MLX5DV_UAR_ALLOC_TYPE_NC
493 * is performed. The drivers specifying negative values should
494 * always provide the write memory barrier operation after UAR
496 * If there is no definitions for the MLX5DV_UAR_ALLOC_TYPE_xx (older rdma
497 * library headers), the caller can specify 0.
500 * UAR object pointer on success, NULL otherwise and rte_errno is set.
503 mlx5_devx_alloc_uar(void *ctx, int mapping)
506 uint32_t retry, uar_mapping;
509 for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) {
510 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC
511 /* Control the mapping type according to the settings. */
512 uar_mapping = (mapping < 0) ?
513 MLX5DV_UAR_ALLOC_TYPE_NC : mapping;
516 * It seems we have no way to control the memory mapping type
517 * for the UAR, the default "Write-Combining" type is supposed.
520 RTE_SET_USED(mapping);
522 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping);
523 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC
526 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) {
528 * In some environments like virtual machine the
529 * Write Combining mapped might be not supported and
530 * UAR allocation fails. We tried "Non-Cached" mapping
533 DRV_LOG(WARNING, "Failed to allocate DevX UAR (BF)");
534 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC;
535 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping);
538 uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) {
540 * If Verbs/kernel does not support "Non-Cached"
541 * try the "Write-Combining".
543 DRV_LOG(WARNING, "Failed to allocate DevX UAR (NC)");
544 uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF;
545 uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping);
549 DRV_LOG(ERR, "Failed to allocate DevX UAR (BF/NC)");
553 base_addr = mlx5_os_get_devx_uar_base_addr(uar);
557 * The UARs are allocated by rdma_core within the
558 * IB device context, on context closure all UARs
559 * will be freed, should be no memory/object leakage.
561 DRV_LOG(WARNING, "Retrying to allocate DevX UAR");
564 /* Check whether we finally succeeded with valid UAR allocation. */
566 DRV_LOG(ERR, "Failed to allocate DevX UAR (NULL base)");
570 * Return void * instead of struct mlx5dv_devx_uar *
571 * is for compatibility with older rdma-core library headers.
577 RTE_PMD_EXPORT_NAME(mlx5_common_driver, __COUNTER__);