1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
7 #include <rte_common.h>
8 #include <rte_eal_paging.h>
10 #include <mlx5_glue.h>
11 #include <mlx5_common_os.h>
14 #include "mlx5_devx_cmds.h"
15 #include "mlx5_common_log.h"
16 #include "mlx5_malloc.h"
17 #include "mlx5_common.h"
18 #include "mlx5_common_devx.h"
21 * Destroy DevX Completion Queue.
27 mlx5_devx_cq_destroy(struct mlx5_devx_cq *cq)
30 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
32 claim_zero(mlx5_os_umem_dereg(cq->umem_obj));
34 mlx5_free((void *)(uintptr_t)cq->umem_buf);
37 /* Mark all CQEs initially as invalid. */
39 mlx5_cq_init(struct mlx5_devx_cq *cq_obj, uint16_t cq_size)
41 volatile struct mlx5_cqe *cqe = cq_obj->cqes;
44 for (i = 0; i < cq_size; i++, cqe++)
45 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK;
49 * Create Completion Queue using DevX API.
51 * Get a pointer to partially initialized attributes structure, and updates the
62 * All other fields are updated by caller.
65 * Context returned from mlx5 open_device() glue function.
66 * @param[in/out] cq_obj
67 * Pointer to CQ to create.
68 * @param[in] log_desc_n
69 * Log of number of descriptors in queue.
71 * Pointer to CQ attributes structure.
73 * Socket to use for allocation.
76 * 0 on success, a negative errno value otherwise and rte_errno is set.
79 mlx5_devx_cq_create(void *ctx, struct mlx5_devx_cq *cq_obj, uint16_t log_desc_n,
80 struct mlx5_devx_cq_attr *attr, int socket)
82 struct mlx5_devx_obj *cq = NULL;
83 struct mlx5dv_devx_umem *umem_obj = NULL;
84 void *umem_buf = NULL;
85 size_t page_size = rte_mem_page_size();
86 size_t alignment = MLX5_CQE_BUF_ALIGNMENT;
87 uint32_t umem_size, umem_dbrec;
89 uint16_t cq_size = 1 << log_desc_n;
92 if (page_size == (size_t)-1 || alignment == (size_t)-1) {
93 DRV_LOG(ERR, "Failed to get page_size.");
97 /* Query first EQN. */
98 ret = mlx5_glue->devx_query_eqn(ctx, 0, &eqn);
101 DRV_LOG(ERR, "Failed to query event queue number.");
104 /* Allocate memory buffer for CQEs and doorbell record. */
105 umem_size = sizeof(struct mlx5_cqe) * cq_size;
106 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
107 umem_size += MLX5_DBR_SIZE;
108 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
111 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
115 /* Register allocated buffer in user space with DevX. */
116 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
117 IBV_ACCESS_LOCAL_WRITE);
119 DRV_LOG(ERR, "Failed to register umem for CQ.");
123 /* Fill attributes for CQ object creation. */
124 attr->q_umem_valid = 1;
125 attr->q_umem_id = mlx5_os_get_umem_id(umem_obj);
126 attr->q_umem_offset = 0;
127 attr->db_umem_valid = 1;
128 attr->db_umem_id = attr->q_umem_id;
129 attr->db_umem_offset = umem_dbrec;
131 attr->log_cq_size = log_desc_n;
132 attr->log_page_size = rte_log2_u32(page_size);
133 /* Create completion queue object with DevX. */
134 cq = mlx5_devx_cmd_create_cq(ctx, attr);
136 DRV_LOG(ERR, "Can't create DevX CQ object.");
140 cq_obj->umem_buf = umem_buf;
141 cq_obj->umem_obj = umem_obj;
143 cq_obj->db_rec = RTE_PTR_ADD(cq_obj->umem_buf, umem_dbrec);
144 /* Mark all CQEs initially as invalid. */
145 mlx5_cq_init(cq_obj, cq_size);
150 claim_zero(mlx5_os_umem_dereg(umem_obj));
152 mlx5_free((void *)(uintptr_t)umem_buf);
158 * Destroy DevX Send Queue.
161 * DevX SQ to destroy.
164 mlx5_devx_sq_destroy(struct mlx5_devx_sq *sq)
167 claim_zero(mlx5_devx_cmd_destroy(sq->sq));
169 claim_zero(mlx5_os_umem_dereg(sq->umem_obj));
171 mlx5_free((void *)(uintptr_t)sq->umem_buf);
175 * Create Send Queue using DevX API.
177 * Get a pointer to partially initialized attributes structure, and updates the
189 * All other fields are updated by caller.
192 * Context returned from mlx5 open_device() glue function.
193 * @param[in/out] sq_obj
194 * Pointer to SQ to create.
195 * @param[in] log_wqbb_n
196 * Log of number of WQBBs in queue.
198 * Pointer to SQ attributes structure.
200 * Socket to use for allocation.
203 * 0 on success, a negative errno value otherwise and rte_errno is set.
206 mlx5_devx_sq_create(void *ctx, struct mlx5_devx_sq *sq_obj, uint16_t log_wqbb_n,
207 struct mlx5_devx_create_sq_attr *attr, int socket)
209 struct mlx5_devx_obj *sq = NULL;
210 struct mlx5dv_devx_umem *umem_obj = NULL;
211 void *umem_buf = NULL;
212 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
213 uint32_t umem_size, umem_dbrec;
214 uint16_t sq_size = 1 << log_wqbb_n;
217 if (alignment == (size_t)-1) {
218 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
222 /* Allocate memory buffer for WQEs and doorbell record. */
223 umem_size = MLX5_WQE_SIZE * sq_size;
224 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
225 umem_size += MLX5_DBR_SIZE;
226 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
229 DRV_LOG(ERR, "Failed to allocate memory for SQ.");
233 /* Register allocated buffer in user space with DevX. */
234 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
235 IBV_ACCESS_LOCAL_WRITE);
237 DRV_LOG(ERR, "Failed to register umem for SQ.");
241 /* Fill attributes for SQ object creation. */
242 attr->wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;
243 attr->wq_attr.wq_umem_valid = 1;
244 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj);
245 attr->wq_attr.wq_umem_offset = 0;
246 attr->wq_attr.dbr_umem_valid = 1;
247 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id;
248 attr->wq_attr.dbr_addr = umem_dbrec;
249 attr->wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE);
250 attr->wq_attr.log_wq_sz = log_wqbb_n;
251 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
252 /* Create send queue object with DevX. */
253 sq = mlx5_devx_cmd_create_sq(ctx, attr);
255 DRV_LOG(ERR, "Can't create DevX SQ object.");
259 sq_obj->umem_buf = umem_buf;
260 sq_obj->umem_obj = umem_obj;
262 sq_obj->db_rec = RTE_PTR_ADD(sq_obj->umem_buf, umem_dbrec);
267 claim_zero(mlx5_os_umem_dereg(umem_obj));
269 mlx5_free((void *)(uintptr_t)umem_buf);
275 * Destroy DevX Queue Pair.
278 * DevX QP to destroy.
281 mlx5_devx_qp_destroy(struct mlx5_devx_qp *qp)
284 claim_zero(mlx5_devx_cmd_destroy(qp->qp));
286 claim_zero(mlx5_os_umem_dereg(qp->umem_obj));
288 mlx5_free((void *)(uintptr_t)qp->umem_buf);
292 * Create Queue Pair using DevX API.
294 * Get a pointer to partially initialized attributes structure, and updates the
302 * All other fields are updated by caller.
305 * Context returned from mlx5 open_device() glue function.
306 * @param[in/out] qp_obj
307 * Pointer to QP to create.
308 * @param[in] log_wqbb_n
309 * Log of number of WQBBs in queue.
311 * Pointer to QP attributes structure.
313 * Socket to use for allocation.
316 * 0 on success, a negative errno value otherwise and rte_errno is set.
319 mlx5_devx_qp_create(void *ctx, struct mlx5_devx_qp *qp_obj, uint16_t log_wqbb_n,
320 struct mlx5_devx_qp_attr *attr, int socket)
322 struct mlx5_devx_obj *qp = NULL;
323 struct mlx5dv_devx_umem *umem_obj = NULL;
324 void *umem_buf = NULL;
325 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
326 uint32_t umem_size, umem_dbrec;
327 uint16_t qp_size = 1 << log_wqbb_n;
330 if (alignment == (size_t)-1) {
331 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
335 /* Allocate memory buffer for WQEs and doorbell record. */
336 umem_size = MLX5_WQE_SIZE * qp_size;
337 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
338 umem_size += MLX5_DBR_SIZE;
339 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
342 DRV_LOG(ERR, "Failed to allocate memory for QP.");
346 /* Register allocated buffer in user space with DevX. */
347 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
348 IBV_ACCESS_LOCAL_WRITE);
350 DRV_LOG(ERR, "Failed to register umem for QP.");
354 /* Fill attributes for SQ object creation. */
355 attr->wq_umem_id = mlx5_os_get_umem_id(umem_obj);
356 attr->wq_umem_offset = 0;
357 attr->dbr_umem_valid = 1;
358 attr->dbr_umem_id = attr->wq_umem_id;
359 attr->dbr_address = umem_dbrec;
360 attr->log_page_size = MLX5_LOG_PAGE_SIZE;
361 /* Create send queue object with DevX. */
362 qp = mlx5_devx_cmd_create_qp(ctx, attr);
364 DRV_LOG(ERR, "Can't create DevX QP object.");
368 qp_obj->umem_buf = umem_buf;
369 qp_obj->umem_obj = umem_obj;
371 qp_obj->db_rec = RTE_PTR_ADD(qp_obj->umem_buf, umem_dbrec);
376 claim_zero(mlx5_os_umem_dereg(umem_obj));
378 mlx5_free((void *)(uintptr_t)umem_buf);
384 * Destroy DevX Receive Queue.
387 * DevX RQ to destroy.
390 mlx5_devx_rq_destroy(struct mlx5_devx_rq *rq)
393 claim_zero(mlx5_devx_cmd_destroy(rq->rq));
395 claim_zero(mlx5_os_umem_dereg(rq->umem_obj));
397 mlx5_free((void *)(uintptr_t)rq->umem_buf);
401 * Create Receive Queue using DevX API.
403 * Get a pointer to partially initialized attributes structure, and updates the
412 * All other fields are updated by caller.
415 * Context returned from mlx5 open_device() glue function.
416 * @param[in/out] rq_obj
417 * Pointer to RQ to create.
418 * @param[in] wqe_size
419 * Size of WQE structure.
420 * @param[in] log_wqbb_n
421 * Log of number of WQBBs in queue.
423 * Pointer to RQ attributes structure.
425 * Socket to use for allocation.
428 * 0 on success, a negative errno value otherwise and rte_errno is set.
431 mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj, uint32_t wqe_size,
433 struct mlx5_devx_create_rq_attr *attr, int socket)
435 struct mlx5_devx_obj *rq = NULL;
436 struct mlx5dv_devx_umem *umem_obj = NULL;
437 void *umem_buf = NULL;
438 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
439 uint32_t umem_size, umem_dbrec;
440 uint16_t rq_size = 1 << log_wqbb_n;
443 if (alignment == (size_t)-1) {
444 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
448 /* Allocate memory buffer for WQEs and doorbell record. */
449 umem_size = wqe_size * rq_size;
450 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
451 umem_size += MLX5_DBR_SIZE;
452 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
455 DRV_LOG(ERR, "Failed to allocate memory for RQ.");
459 /* Register allocated buffer in user space with DevX. */
460 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf,
463 DRV_LOG(ERR, "Failed to register umem for RQ.");
467 /* Fill attributes for RQ object creation. */
468 attr->wq_attr.wq_umem_valid = 1;
469 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj);
470 attr->wq_attr.wq_umem_offset = 0;
471 attr->wq_attr.dbr_umem_valid = 1;
472 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id;
473 attr->wq_attr.dbr_addr = umem_dbrec;
474 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
475 /* Create receive queue object with DevX. */
476 rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
478 DRV_LOG(ERR, "Can't create DevX RQ object.");
482 rq_obj->umem_buf = umem_buf;
483 rq_obj->umem_obj = umem_obj;
485 rq_obj->db_rec = RTE_PTR_ADD(rq_obj->umem_buf, umem_dbrec);
490 claim_zero(mlx5_os_umem_dereg(umem_obj));
492 mlx5_free((void *)(uintptr_t)umem_buf);
499 * Change QP state to RTS.
503 * @param[in] remote_qp_id
504 * The remote QP ID for MLX5_CMD_OP_INIT2RTR_QP operation.
507 * 0 on success, a negative errno value otherwise and rte_errno is set.
510 mlx5_devx_qp2rts(struct mlx5_devx_qp *qp, uint32_t remote_qp_id)
512 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RST2INIT_QP,
514 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
518 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_INIT2RTR_QP,
520 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
524 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RTR2RTS_QP,
526 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",