1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
9 #include <rte_mempool.h>
10 #include <rte_bus_pci.h>
11 #include <rte_malloc.h>
12 #include <rte_errno.h>
14 #include "mlx5_devx_cmds.h"
15 #include "../mlx5_common_log.h"
16 #include "mlx5_common.h"
17 #include "mlx5_common_os.h"
18 #include "mlx5_malloc.h"
21 * Initialization routine for run-time dependency on external lib.
24 mlx5_glue_constructor(void)
29 * Validate user arguments for remote PD and CTX.
32 * Pointer to device configuration structure.
35 * 0 on success, a negative errno value otherwise and rte_errno is set.
38 mlx5_os_remote_pd_and_ctx_validate(struct mlx5_common_dev_config *config)
40 int device_fd = config->device_fd;
41 int pd_handle = config->pd_handle;
43 if (pd_handle != MLX5_ARG_UNSET || device_fd != MLX5_ARG_UNSET) {
44 DRV_LOG(ERR, "Remote PD and CTX is not supported on Windows.");
52 * Release PD. Releases a given mlx5_pd object
55 * Pointer to the mlx5 device.
58 * Zero if pd is released successfully, negative number otherwise.
61 mlx5_os_pd_release(struct mlx5_common_device *cdev)
63 struct mlx5_pd *pd = cdev->pd;
67 mlx5_devx_cmd_destroy(pd->obj);
73 * Allocate Protection Domain object and extract its pdn using DV API.
76 * Pointer to the mlx5 device.
79 * 0 on success, a negative value otherwise.
82 mlx5_os_pd_prepare(struct mlx5_common_device *cdev)
86 pd = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pd), 0, SOCKET_ID_ANY);
89 struct mlx5_devx_obj *obj = mlx5_devx_cmd_alloc_pd(cdev->ctx);
96 pd->devx_ctx = cdev->ctx;
103 * Detect if a devx_device_bdf object has identical DBDF values to the
104 * rte_pci_addr found in bus/pci probing.
106 * @param[in] devx_bdf
107 * Pointer to the devx_device_bdf structure.
109 * Pointer to the rte_pci_addr structure.
112 * 1 on Device match, 0 on mismatch.
115 mlx5_match_devx_bdf_to_addr(struct devx_device_bdf *devx_bdf,
116 struct rte_pci_addr *addr)
118 if (addr->domain != (devx_bdf->bus_id >> 8) ||
119 addr->bus != (devx_bdf->bus_id & 0xff) ||
120 addr->devid != devx_bdf->dev_id ||
121 addr->function != devx_bdf->fnc_id) {
128 * Detect if a devx_device_bdf object matches the rte_pci_addr
129 * found in bus/pci probing
130 * Compare both the Native/PF BDF and the raw_bdf representing a VF BDF.
132 * @param[in] devx_bdf
133 * Pointer to the devx_device_bdf structure.
135 * Pointer to the rte_pci_addr structure.
138 * 1 on Device match, 0 on mismatch, rte_errno code on failure.
141 mlx5_match_devx_devices_to_addr(struct devx_device_bdf *devx_bdf,
142 struct rte_pci_addr *addr)
145 struct devx_device mlx5_dev;
147 if (mlx5_match_devx_bdf_to_addr(devx_bdf, addr))
150 * Didn't match on Native/PF BDF, could still match a VF BDF,
153 err = mlx5_glue->query_device(devx_bdf, &mlx5_dev);
155 DRV_LOG(ERR, "query_device failed");
159 if (mlx5_match_devx_bdf_to_addr(&mlx5_dev.raw_bdf, addr))
165 * Look for DevX device that match to given rte_device.
168 * Pointer to the generic device.
170 * Pointer to head of DevX devices list.
172 * Number of devices in given DevX devices list.
175 * A device match on success, NULL otherwise and rte_errno is set.
177 static struct devx_device_bdf *
178 mlx5_os_get_devx_device(struct rte_device *dev,
179 struct devx_device_bdf *devx_list, int n)
181 struct devx_device_bdf *devx_match = NULL;
182 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev);
183 struct rte_pci_addr *addr = &pci_dev->addr;
186 int ret = mlx5_match_devx_devices_to_addr(devx_list, addr);
195 devx_match = devx_list;
198 if (devx_match == NULL) {
199 /* No device matches, just complain and bail out. */
201 "No DevX device matches PCI device " PCI_PRI_FMT ","
202 " is DevX Configured?",
203 addr->domain, addr->bus, addr->devid, addr->function);
210 * Function API open device under Windows.
212 * This function calls the Windows glue APIs to open a device.
215 * Pointer to mlx5 device structure.
217 * Chosen classes come from user device arguments.
220 * 0 on success, a negative errno value otherwise and rte_errno is set.
223 mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes)
225 struct devx_device_bdf *devx_bdf_dev = NULL;
226 struct devx_device_bdf *devx_list;
227 struct mlx5_context *mlx5_ctx = NULL;
230 if (classes != MLX5_CLASS_ETH && classes != MLX5_CLASS_CRYPTO) {
232 "The chosen classes are not supported on Windows.");
237 devx_list = mlx5_glue->get_device_list(&n);
238 if (devx_list == NULL) {
239 rte_errno = errno ? errno : ENOSYS;
240 DRV_LOG(ERR, "Cannot list devices, is DevX enabled?");
243 devx_bdf_dev = mlx5_os_get_devx_device(cdev->dev, devx_list, n);
244 if (devx_bdf_dev == NULL)
246 /* Try to open DevX device with DV. */
247 mlx5_ctx = mlx5_glue->open_device(devx_bdf_dev);
248 if (mlx5_ctx == NULL) {
249 DRV_LOG(ERR, "Failed to open DevX device.");
253 if (mlx5_glue->query_device(devx_bdf_dev, &mlx5_ctx->mlx5_dev)) {
254 DRV_LOG(ERR, "Failed to query device context fields.");
258 cdev->config.devx = 1;
259 cdev->ctx = mlx5_ctx;
260 mlx5_glue->free_device_list(devx_list);
263 if (mlx5_ctx != NULL)
264 claim_zero(mlx5_glue->close_device(mlx5_ctx));
265 mlx5_glue->free_device_list(devx_list);
273 * Pointer to context.
275 * Pointer to memory start address.
277 * Size of the memory to register.
282 * umem on successful registration, NULL and errno otherwise
285 mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access)
287 struct mlx5_devx_umem *umem;
289 umem = mlx5_malloc(MLX5_MEM_ZERO,
290 (sizeof(*umem)), 0, SOCKET_ID_ANY);
295 umem->umem_hdl = mlx5_glue->devx_umem_reg(ctx, addr, size, access,
297 if (!umem->umem_hdl) {
312 * 0 on successful release, negative number otherwise
315 mlx5_os_umem_dereg(void *pumem)
317 struct mlx5_devx_umem *umem;
324 err = mlx5_glue->devx_umem_dereg(umem->umem_hdl);
330 * Register mr. Given protection domain pointer, pointer to addr and length
331 * register the memory region.
334 * Pointer to protection domain context (type mlx5_pd).
336 * Pointer to memory start address (type devx_device_ctx).
338 * Length of the memory to register.
340 * pmd_mr struct set with lkey, address, length, pointer to mr object, mkey
343 * 0 on successful registration, -1 otherwise
346 mlx5_os_reg_mr(void *pd,
347 void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr)
349 struct mlx5_devx_mkey_attr mkey_attr;
350 struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd;
351 struct mlx5_hca_attr attr;
352 struct mlx5_devx_obj *mkey;
359 if (mlx5_devx_cmd_query_hca_attr(mlx5_pd->devx_ctx, &attr))
361 obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, addr, length,
362 IBV_ACCESS_LOCAL_WRITE);
365 memset(&mkey_attr, 0, sizeof(mkey_attr));
366 mkey_attr.addr = (uintptr_t)addr;
367 mkey_attr.size = length;
368 mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id;
369 mkey_attr.pd = mlx5_pd->pdn;
370 if (!haswell_broadwell_cpu) {
371 mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write;
372 mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read;
374 mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);
376 claim_zero(mlx5_os_umem_dereg(obj));
380 pmd_mr->len = length;
383 pmd_mr->lkey = pmd_mr->mkey->id;
391 * Pointer to PMD mr object
394 mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
399 claim_zero(mlx5_devx_cmd_destroy(pmd_mr->mkey));
401 claim_zero(mlx5_os_umem_dereg(pmd_mr->obj));
402 memset(pmd_mr, 0, sizeof(*pmd_mr));
406 * Set the reg_mr and dereg_mr callbacks.
408 * @param[out] reg_mr_cb
409 * Pointer to reg_mr func
410 * @param[out] dereg_mr_cb
411 * Pointer to dereg_mr func
415 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
417 *reg_mr_cb = mlx5_os_reg_mr;
418 *dereg_mr_cb = mlx5_os_dereg_mr;
422 * In Windows, no need to wrap the MR, no known issue for it in kernel.
423 * Use the regular function to create direct MR.
426 mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr,
427 size_t length, struct mlx5_pmd_wrapped_mr *wpmd_mr)
429 struct mlx5_pmd_mr pmd_mr = {0};
430 int ret = mlx5_os_reg_mr(pd, addr, length, &pmd_mr);
436 wpmd_mr->addr = addr;
437 wpmd_mr->len = length;
438 wpmd_mr->obj = pmd_mr.obj;
439 wpmd_mr->imkey = pmd_mr.mkey;
440 wpmd_mr->lkey = pmd_mr.mkey->id;
445 mlx5_os_wrapped_mkey_destroy(struct mlx5_pmd_wrapped_mr *wpmd_mr)
447 struct mlx5_pmd_mr pmd_mr;
451 pmd_mr.addr = wpmd_mr->addr;
452 pmd_mr.len = wpmd_mr->len;
453 pmd_mr.obj = wpmd_mr->obj;
454 pmd_mr.mkey = wpmd_mr->imkey;
455 pmd_mr.lkey = wpmd_mr->lkey;
456 mlx5_os_dereg_mr(&pmd_mr);
457 memset(wpmd_mr, 0, sizeof(*wpmd_mr));