1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
7 #include <rte_common.h>
8 #include <rte_eal_paging.h>
10 #include <mlx5_glue.h>
11 #include <mlx5_common_os.h>
14 #include "mlx5_devx_cmds.h"
15 #include "mlx5_common_utils.h"
16 #include "mlx5_malloc.h"
17 #include "mlx5_common.h"
18 #include "mlx5_common_devx.h"
21 * Destroy DevX Completion Queue.
27 mlx5_devx_cq_destroy(struct mlx5_devx_cq *cq)
30 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
32 claim_zero(mlx5_os_umem_dereg(cq->umem_obj));
34 mlx5_free((void *)(uintptr_t)cq->umem_buf);
37 /* Mark all CQEs initially as invalid. */
39 mlx5_cq_init(struct mlx5_devx_cq *cq_obj, uint16_t cq_size)
41 volatile struct mlx5_cqe *cqe = cq_obj->cqes;
44 for (i = 0; i < cq_size; i++, cqe++)
45 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK;
49 * Create Completion Queue using DevX API.
51 * Get a pointer to partially initialized attributes structure, and updates the
62 * All other fields are updated by caller.
65 * Context returned from mlx5 open_device() glue function.
66 * @param[in/out] cq_obj
67 * Pointer to CQ to create.
68 * @param[in] log_desc_n
69 * Log of number of descriptors in queue.
71 * Pointer to CQ attributes structure.
73 * Socket to use for allocation.
76 * 0 on success, a negative errno value otherwise and rte_errno is set.
79 mlx5_devx_cq_create(void *ctx, struct mlx5_devx_cq *cq_obj, uint16_t log_desc_n,
80 struct mlx5_devx_cq_attr *attr, int socket)
82 struct mlx5_devx_obj *cq = NULL;
83 struct mlx5dv_devx_umem *umem_obj = NULL;
84 void *umem_buf = NULL;
85 size_t page_size = rte_mem_page_size();
86 size_t alignment = MLX5_CQE_BUF_ALIGNMENT;
87 uint32_t umem_size, umem_dbrec;
89 uint16_t cq_size = 1 << log_desc_n;
92 if (page_size == (size_t)-1 || alignment == (size_t)-1) {
93 DRV_LOG(ERR, "Failed to get page_size.");
97 /* Query first EQN. */
98 ret = mlx5_glue->devx_query_eqn(ctx, 0, &eqn);
101 DRV_LOG(ERR, "Failed to query event queue number.");
104 /* Allocate memory buffer for CQEs and doorbell record. */
105 umem_size = sizeof(struct mlx5_cqe) * cq_size;
106 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
107 umem_size += MLX5_DBR_SIZE;
108 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
111 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
115 /* Register allocated buffer in user space with DevX. */
116 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
117 IBV_ACCESS_LOCAL_WRITE);
119 DRV_LOG(ERR, "Failed to register umem for CQ.");
123 /* Fill attributes for CQ object creation. */
124 attr->q_umem_valid = 1;
125 attr->q_umem_id = mlx5_os_get_umem_id(umem_obj);
126 attr->q_umem_offset = 0;
127 attr->db_umem_valid = 1;
128 attr->db_umem_id = attr->q_umem_id;
129 attr->db_umem_offset = umem_dbrec;
131 attr->log_cq_size = log_desc_n;
132 attr->log_page_size = rte_log2_u32(page_size);
133 /* Create completion queue object with DevX. */
134 cq = mlx5_devx_cmd_create_cq(ctx, attr);
136 DRV_LOG(ERR, "Can't create DevX CQ object.");
140 cq_obj->umem_buf = umem_buf;
141 cq_obj->umem_obj = umem_obj;
143 cq_obj->db_rec = RTE_PTR_ADD(cq_obj->umem_buf, umem_dbrec);
144 /* Mark all CQEs initially as invalid. */
145 mlx5_cq_init(cq_obj, cq_size);
150 claim_zero(mlx5_os_umem_dereg(umem_obj));
152 mlx5_free((void *)(uintptr_t)umem_buf);
158 * Destroy DevX Send Queue.
161 * DevX SQ to destroy.
164 mlx5_devx_sq_destroy(struct mlx5_devx_sq *sq)
167 claim_zero(mlx5_devx_cmd_destroy(sq->sq));
169 claim_zero(mlx5_os_umem_dereg(sq->umem_obj));
171 mlx5_free((void *)(uintptr_t)sq->umem_buf);
175 * Create Send Queue using DevX API.
177 * Get a pointer to partially initialized attributes structure, and updates the
189 * All other fields are updated by caller.
192 * Context returned from mlx5 open_device() glue function.
193 * @param[in/out] sq_obj
194 * Pointer to SQ to create.
195 * @param[in] log_wqbb_n
196 * Log of number of WQBBs in queue.
198 * Pointer to SQ attributes structure.
200 * Socket to use for allocation.
203 * 0 on success, a negative errno value otherwise and rte_errno is set.
206 mlx5_devx_sq_create(void *ctx, struct mlx5_devx_sq *sq_obj, uint16_t log_wqbb_n,
207 struct mlx5_devx_create_sq_attr *attr, int socket)
209 struct mlx5_devx_obj *sq = NULL;
210 struct mlx5dv_devx_umem *umem_obj = NULL;
211 void *umem_buf = NULL;
212 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
213 uint32_t umem_size, umem_dbrec;
214 uint16_t sq_size = 1 << log_wqbb_n;
217 if (alignment == (size_t)-1) {
218 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
222 /* Allocate memory buffer for WQEs and doorbell record. */
223 umem_size = MLX5_WQE_SIZE * sq_size;
224 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
225 umem_size += MLX5_DBR_SIZE;
226 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
229 DRV_LOG(ERR, "Failed to allocate memory for SQ.");
233 /* Register allocated buffer in user space with DevX. */
234 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
235 IBV_ACCESS_LOCAL_WRITE);
237 DRV_LOG(ERR, "Failed to register umem for SQ.");
241 /* Fill attributes for SQ object creation. */
242 attr->wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;
243 attr->wq_attr.wq_umem_valid = 1;
244 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj);
245 attr->wq_attr.wq_umem_offset = 0;
246 attr->wq_attr.dbr_umem_valid = 1;
247 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id;
248 attr->wq_attr.dbr_addr = umem_dbrec;
249 attr->wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE);
250 attr->wq_attr.log_wq_sz = log_wqbb_n;
251 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
252 /* Create send queue object with DevX. */
253 sq = mlx5_devx_cmd_create_sq(ctx, attr);
255 DRV_LOG(ERR, "Can't create DevX SQ object.");
259 sq_obj->umem_buf = umem_buf;
260 sq_obj->umem_obj = umem_obj;
262 sq_obj->db_rec = RTE_PTR_ADD(sq_obj->umem_buf, umem_dbrec);
267 claim_zero(mlx5_os_umem_dereg(umem_obj));
269 mlx5_free((void *)(uintptr_t)umem_buf);
275 * Destroy DevX Receive Queue.
278 * DevX RQ to destroy.
281 mlx5_devx_rq_destroy(struct mlx5_devx_rq *rq)
284 claim_zero(mlx5_devx_cmd_destroy(rq->rq));
286 claim_zero(mlx5_os_umem_dereg(rq->umem_obj));
288 mlx5_free((void *)(uintptr_t)rq->umem_buf);
292 * Create Receive Queue using DevX API.
294 * Get a pointer to partially initialized attributes structure, and updates the
303 * All other fields are updated by caller.
306 * Context returned from mlx5 open_device() glue function.
307 * @param[in/out] rq_obj
308 * Pointer to RQ to create.
309 * @param[in] wqe_size
310 * Size of WQE structure.
311 * @param[in] log_wqbb_n
312 * Log of number of WQBBs in queue.
314 * Pointer to RQ attributes structure.
316 * Socket to use for allocation.
319 * 0 on success, a negative errno value otherwise and rte_errno is set.
322 mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj, uint32_t wqe_size,
324 struct mlx5_devx_create_rq_attr *attr, int socket)
326 struct mlx5_devx_obj *rq = NULL;
327 struct mlx5dv_devx_umem *umem_obj = NULL;
328 void *umem_buf = NULL;
329 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
330 uint32_t umem_size, umem_dbrec;
331 uint16_t rq_size = 1 << log_wqbb_n;
334 if (alignment == (size_t)-1) {
335 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
339 /* Allocate memory buffer for WQEs and doorbell record. */
340 umem_size = wqe_size * rq_size;
341 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
342 umem_size += MLX5_DBR_SIZE;
343 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
346 DRV_LOG(ERR, "Failed to allocate memory for RQ.");
350 /* Register allocated buffer in user space with DevX. */
351 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf,
354 DRV_LOG(ERR, "Failed to register umem for RQ.");
358 /* Fill attributes for RQ object creation. */
359 attr->wq_attr.wq_umem_valid = 1;
360 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj);
361 attr->wq_attr.wq_umem_offset = 0;
362 attr->wq_attr.dbr_umem_valid = 1;
363 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id;
364 attr->wq_attr.dbr_addr = umem_dbrec;
365 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
366 /* Create receive queue object with DevX. */
367 rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
369 DRV_LOG(ERR, "Can't create DevX RQ object.");
373 rq_obj->umem_buf = umem_buf;
374 rq_obj->umem_obj = umem_obj;
376 rq_obj->db_rec = RTE_PTR_ADD(rq_obj->umem_buf, umem_dbrec);
381 claim_zero(mlx5_os_umem_dereg(umem_obj));
383 mlx5_free((void *)(uintptr_t)umem_buf);