1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
7 #include <rte_common.h>
8 #include <rte_eal_paging.h>
10 #include <mlx5_glue.h>
11 #include <mlx5_common_os.h>
14 #include "mlx5_devx_cmds.h"
15 #include "mlx5_common_log.h"
16 #include "mlx5_malloc.h"
17 #include "mlx5_common.h"
18 #include "mlx5_common_devx.h"
21 * Destroy DevX Completion Queue.
27 mlx5_devx_cq_destroy(struct mlx5_devx_cq *cq)
30 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
32 claim_zero(mlx5_os_umem_dereg(cq->umem_obj));
34 mlx5_free((void *)(uintptr_t)cq->umem_buf);
37 /* Mark all CQEs initially as invalid. */
39 mlx5_cq_init(struct mlx5_devx_cq *cq_obj, uint16_t cq_size)
41 volatile struct mlx5_cqe *cqe = cq_obj->cqes;
44 for (i = 0; i < cq_size; i++, cqe++)
45 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK;
49 * Create Completion Queue using DevX API.
51 * Get a pointer to partially initialized attributes structure, and updates the
62 * All other fields are updated by caller.
65 * Context returned from mlx5 open_device() glue function.
66 * @param[in/out] cq_obj
67 * Pointer to CQ to create.
68 * @param[in] log_desc_n
69 * Log of number of descriptors in queue.
71 * Pointer to CQ attributes structure.
73 * Socket to use for allocation.
76 * 0 on success, a negative errno value otherwise and rte_errno is set.
79 mlx5_devx_cq_create(void *ctx, struct mlx5_devx_cq *cq_obj, uint16_t log_desc_n,
80 struct mlx5_devx_cq_attr *attr, int socket)
82 struct mlx5_devx_obj *cq = NULL;
83 struct mlx5dv_devx_umem *umem_obj = NULL;
84 void *umem_buf = NULL;
85 size_t page_size = rte_mem_page_size();
86 size_t alignment = MLX5_CQE_BUF_ALIGNMENT;
87 uint32_t umem_size, umem_dbrec;
89 uint32_t num_of_cqes = RTE_BIT32(log_desc_n);
92 if (page_size == (size_t)-1 || alignment == (size_t)-1) {
93 DRV_LOG(ERR, "Failed to get page_size.");
97 /* Query first EQN. */
98 ret = mlx5_glue->devx_query_eqn(ctx, 0, &eqn);
101 DRV_LOG(ERR, "Failed to query event queue number.");
104 /* Allocate memory buffer for CQEs and doorbell record. */
105 umem_size = sizeof(struct mlx5_cqe) * num_of_cqes;
106 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
107 umem_size += MLX5_DBR_SIZE;
108 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
111 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
115 /* Register allocated buffer in user space with DevX. */
116 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
117 IBV_ACCESS_LOCAL_WRITE);
119 DRV_LOG(ERR, "Failed to register umem for CQ.");
123 /* Fill attributes for CQ object creation. */
124 attr->q_umem_valid = 1;
125 attr->q_umem_id = mlx5_os_get_umem_id(umem_obj);
126 attr->q_umem_offset = 0;
127 attr->db_umem_valid = 1;
128 attr->db_umem_id = attr->q_umem_id;
129 attr->db_umem_offset = umem_dbrec;
131 attr->log_cq_size = log_desc_n;
132 attr->log_page_size = rte_log2_u32(page_size);
133 /* Create completion queue object with DevX. */
134 cq = mlx5_devx_cmd_create_cq(ctx, attr);
136 DRV_LOG(ERR, "Can't create DevX CQ object.");
140 cq_obj->umem_buf = umem_buf;
141 cq_obj->umem_obj = umem_obj;
143 cq_obj->db_rec = RTE_PTR_ADD(cq_obj->umem_buf, umem_dbrec);
144 /* Mark all CQEs initially as invalid. */
145 mlx5_cq_init(cq_obj, num_of_cqes);
150 claim_zero(mlx5_os_umem_dereg(umem_obj));
152 mlx5_free((void *)(uintptr_t)umem_buf);
158 * Destroy DevX Send Queue.
161 * DevX SQ to destroy.
164 mlx5_devx_sq_destroy(struct mlx5_devx_sq *sq)
167 claim_zero(mlx5_devx_cmd_destroy(sq->sq));
169 claim_zero(mlx5_os_umem_dereg(sq->umem_obj));
171 mlx5_free((void *)(uintptr_t)sq->umem_buf);
175 * Create Send Queue using DevX API.
177 * Get a pointer to partially initialized attributes structure, and updates the
189 * All other fields are updated by caller.
192 * Context returned from mlx5 open_device() glue function.
193 * @param[in/out] sq_obj
194 * Pointer to SQ to create.
195 * @param[in] log_wqbb_n
196 * Log of number of WQBBs in queue.
198 * Pointer to SQ attributes structure.
200 * Socket to use for allocation.
203 * 0 on success, a negative errno value otherwise and rte_errno is set.
206 mlx5_devx_sq_create(void *ctx, struct mlx5_devx_sq *sq_obj, uint16_t log_wqbb_n,
207 struct mlx5_devx_create_sq_attr *attr, int socket)
209 struct mlx5_devx_obj *sq = NULL;
210 struct mlx5dv_devx_umem *umem_obj = NULL;
211 void *umem_buf = NULL;
212 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
213 uint32_t umem_size, umem_dbrec;
214 uint32_t num_of_wqbbs = RTE_BIT32(log_wqbb_n);
217 if (alignment == (size_t)-1) {
218 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
222 /* Allocate memory buffer for WQEs and doorbell record. */
223 umem_size = MLX5_WQE_SIZE * num_of_wqbbs;
224 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
225 umem_size += MLX5_DBR_SIZE;
226 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
229 DRV_LOG(ERR, "Failed to allocate memory for SQ.");
233 /* Register allocated buffer in user space with DevX. */
234 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
235 IBV_ACCESS_LOCAL_WRITE);
237 DRV_LOG(ERR, "Failed to register umem for SQ.");
241 /* Fill attributes for SQ object creation. */
242 attr->wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;
243 attr->wq_attr.wq_umem_valid = 1;
244 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj);
245 attr->wq_attr.wq_umem_offset = 0;
246 attr->wq_attr.dbr_umem_valid = 1;
247 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id;
248 attr->wq_attr.dbr_addr = umem_dbrec;
249 attr->wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE);
250 attr->wq_attr.log_wq_sz = log_wqbb_n;
251 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
252 /* Create send queue object with DevX. */
253 sq = mlx5_devx_cmd_create_sq(ctx, attr);
255 DRV_LOG(ERR, "Can't create DevX SQ object.");
259 sq_obj->umem_buf = umem_buf;
260 sq_obj->umem_obj = umem_obj;
262 sq_obj->db_rec = RTE_PTR_ADD(sq_obj->umem_buf, umem_dbrec);
267 claim_zero(mlx5_os_umem_dereg(umem_obj));
269 mlx5_free((void *)(uintptr_t)umem_buf);
275 * Destroy DevX Receive Queue resources.
278 * DevX RQ resource to destroy.
281 mlx5_devx_wq_res_destroy(struct mlx5_devx_wq_res *rq_res)
283 if (rq_res->umem_obj)
284 claim_zero(mlx5_os_umem_dereg(rq_res->umem_obj));
285 if (rq_res->umem_buf)
286 mlx5_free((void *)(uintptr_t)rq_res->umem_buf);
287 memset(rq_res, 0, sizeof(*rq_res));
291 * Destroy DevX Receive Memory Pool.
294 * DevX RMP to destroy.
297 mlx5_devx_rmp_destroy(struct mlx5_devx_rmp *rmp)
299 MLX5_ASSERT(rmp->ref_cnt == 0);
301 claim_zero(mlx5_devx_cmd_destroy(rmp->rmp));
304 mlx5_devx_wq_res_destroy(&rmp->wq);
308 * Destroy DevX Queue Pair.
311 * DevX QP to destroy.
314 mlx5_devx_qp_destroy(struct mlx5_devx_qp *qp)
317 claim_zero(mlx5_devx_cmd_destroy(qp->qp));
319 claim_zero(mlx5_os_umem_dereg(qp->umem_obj));
321 mlx5_free((void *)(uintptr_t)qp->umem_buf);
325 * Create Queue Pair using DevX API.
327 * Get a pointer to partially initialized attributes structure, and updates the
335 * All other fields are updated by caller.
338 * Context returned from mlx5 open_device() glue function.
339 * @param[in/out] qp_obj
340 * Pointer to QP to create.
341 * @param[in] queue_size
342 * Size of queue to create.
344 * Pointer to QP attributes structure.
346 * Socket to use for allocation.
349 * 0 on success, a negative errno value otherwise and rte_errno is set.
352 mlx5_devx_qp_create(void *ctx, struct mlx5_devx_qp *qp_obj, uint32_t queue_size,
353 struct mlx5_devx_qp_attr *attr, int socket)
355 struct mlx5_devx_obj *qp = NULL;
356 struct mlx5dv_devx_umem *umem_obj = NULL;
357 void *umem_buf = NULL;
358 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
359 uint32_t umem_size, umem_dbrec;
362 if (alignment == (size_t)-1) {
363 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
367 /* Allocate memory buffer for WQEs and doorbell record. */
368 umem_size = queue_size;
369 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
370 umem_size += MLX5_DBR_SIZE;
371 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
374 DRV_LOG(ERR, "Failed to allocate memory for QP.");
378 /* Register allocated buffer in user space with DevX. */
379 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
380 IBV_ACCESS_LOCAL_WRITE);
382 DRV_LOG(ERR, "Failed to register umem for QP.");
386 /* Fill attributes for SQ object creation. */
387 attr->wq_umem_id = mlx5_os_get_umem_id(umem_obj);
388 attr->wq_umem_offset = 0;
389 attr->dbr_umem_valid = 1;
390 attr->dbr_umem_id = attr->wq_umem_id;
391 attr->dbr_address = umem_dbrec;
392 attr->log_page_size = MLX5_LOG_PAGE_SIZE;
393 /* Create send queue object with DevX. */
394 qp = mlx5_devx_cmd_create_qp(ctx, attr);
396 DRV_LOG(ERR, "Can't create DevX QP object.");
400 qp_obj->umem_buf = umem_buf;
401 qp_obj->umem_obj = umem_obj;
403 qp_obj->db_rec = RTE_PTR_ADD(qp_obj->umem_buf, umem_dbrec);
408 claim_zero(mlx5_os_umem_dereg(umem_obj));
410 mlx5_free((void *)(uintptr_t)umem_buf);
416 * Destroy DevX Receive Queue.
419 * DevX RQ to destroy.
422 mlx5_devx_rq_destroy(struct mlx5_devx_rq *rq)
425 claim_zero(mlx5_devx_cmd_destroy(rq->rq));
430 if (rq->rmp == NULL) {
431 mlx5_devx_wq_res_destroy(&rq->wq);
433 if (rq->rmp->ref_cnt == 0)
434 mlx5_devx_rmp_destroy(rq->rmp);
439 * Create WQ resources using DevX API.
442 * Context returned from mlx5 open_device() glue function.
443 * @param[in] wqe_size
444 * Size of WQE structure.
445 * @param[in] log_wqbb_n
446 * Log of number of WQBBs in queue.
448 * Socket to use for allocation.
449 * @param[out] wq_attr
450 * Pointer to WQ attributes structure.
452 * Pointer to WQ resource to create.
455 * 0 on success, a negative errno value otherwise and rte_errno is set.
458 mlx5_devx_wq_init(void *ctx, uint32_t wqe_size, uint16_t log_wqbb_n, int socket,
459 struct mlx5_devx_wq_attr *wq_attr,
460 struct mlx5_devx_wq_res *wq_res)
462 struct mlx5dv_devx_umem *umem_obj = NULL;
463 void *umem_buf = NULL;
464 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
465 uint32_t umem_size, umem_dbrec;
468 if (alignment == (size_t)-1) {
469 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
473 /* Allocate memory buffer for WQEs and doorbell record. */
474 umem_size = wqe_size * (1 << log_wqbb_n);
475 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
476 umem_size += MLX5_DBR_SIZE;
477 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
480 DRV_LOG(ERR, "Failed to allocate memory for RQ.");
484 /* Register allocated buffer in user space with DevX. */
485 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf,
488 DRV_LOG(ERR, "Failed to register umem for RQ.");
492 /* Fill WQ attributes for RQ/RMP object creation. */
493 wq_attr->wq_umem_valid = 1;
494 wq_attr->wq_umem_id = mlx5_os_get_umem_id(umem_obj);
495 wq_attr->wq_umem_offset = 0;
496 wq_attr->dbr_umem_valid = 1;
497 wq_attr->dbr_umem_id = wq_attr->wq_umem_id;
498 wq_attr->dbr_addr = umem_dbrec;
499 wq_attr->log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
500 /* Fill attributes for RQ object creation. */
501 wq_res->umem_buf = umem_buf;
502 wq_res->umem_obj = umem_obj;
503 wq_res->db_rec = RTE_PTR_ADD(umem_buf, umem_dbrec);
508 claim_zero(mlx5_os_umem_dereg(umem_obj));
510 mlx5_free((void *)(uintptr_t)umem_buf);
516 * Create standalone Receive Queue using DevX API.
519 * Context returned from mlx5 open_device() glue function.
520 * @param[in/out] rq_obj
521 * Pointer to RQ to create.
522 * @param[in] wqe_size
523 * Size of WQE structure.
524 * @param[in] log_wqbb_n
525 * Log of number of WQBBs in queue.
527 * Pointer to RQ attributes structure.
529 * Socket to use for allocation.
532 * 0 on success, a negative errno value otherwise and rte_errno is set.
535 mlx5_devx_rq_std_create(void *ctx, struct mlx5_devx_rq *rq_obj,
536 uint32_t wqe_size, uint16_t log_wqbb_n,
537 struct mlx5_devx_create_rq_attr *attr, int socket)
539 struct mlx5_devx_obj *rq;
542 ret = mlx5_devx_wq_init(ctx, wqe_size, log_wqbb_n, socket,
543 &attr->wq_attr, &rq_obj->wq);
546 /* Create receive queue object with DevX. */
547 rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
549 DRV_LOG(ERR, "Can't create DevX RQ object.");
557 mlx5_devx_wq_res_destroy(&rq_obj->wq);
563 * Create Receive Memory Pool using DevX API.
566 * Context returned from mlx5 open_device() glue function.
567 * @param[in/out] rq_obj
568 * Pointer to RQ to create.
569 * @param[in] wqe_size
570 * Size of WQE structure.
571 * @param[in] log_wqbb_n
572 * Log of number of WQBBs in queue.
574 * Pointer to RQ attributes structure.
576 * Socket to use for allocation.
579 * 0 on success, a negative errno value otherwise and rte_errno is set.
582 mlx5_devx_rmp_create(void *ctx, struct mlx5_devx_rmp *rmp_obj,
583 uint32_t wqe_size, uint16_t log_wqbb_n,
584 struct mlx5_devx_wq_attr *wq_attr, int socket)
586 struct mlx5_devx_create_rmp_attr rmp_attr = { 0 };
589 if (rmp_obj->rmp != NULL)
591 rmp_attr.wq_attr = *wq_attr;
592 ret = mlx5_devx_wq_init(ctx, wqe_size, log_wqbb_n, socket,
593 &rmp_attr.wq_attr, &rmp_obj->wq);
596 rmp_attr.state = MLX5_RMPC_STATE_RDY;
597 rmp_attr.basic_cyclic_rcv_wqe =
598 wq_attr->wq_type != MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ;
599 /* Create receive memory pool object with DevX. */
600 rmp_obj->rmp = mlx5_devx_cmd_create_rmp(ctx, &rmp_attr, socket);
601 if (rmp_obj->rmp == NULL) {
602 DRV_LOG(ERR, "Can't create DevX RMP object.");
609 mlx5_devx_wq_res_destroy(&rmp_obj->wq);
615 * Create Shared Receive Queue based on RMP using DevX API.
618 * Context returned from mlx5 open_device() glue function.
619 * @param[in/out] rq_obj
620 * Pointer to RQ to create.
621 * @param[in] wqe_size
622 * Size of WQE structure.
623 * @param[in] log_wqbb_n
624 * Log of number of WQBBs in queue.
626 * Pointer to RQ attributes structure.
628 * Socket to use for allocation.
631 * 0 on success, a negative errno value otherwise and rte_errno is set.
634 mlx5_devx_rq_shared_create(void *ctx, struct mlx5_devx_rq *rq_obj,
635 uint32_t wqe_size, uint16_t log_wqbb_n,
636 struct mlx5_devx_create_rq_attr *attr, int socket)
638 struct mlx5_devx_obj *rq;
641 ret = mlx5_devx_rmp_create(ctx, rq_obj->rmp, wqe_size, log_wqbb_n,
642 &attr->wq_attr, socket);
645 attr->mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_RMP;
646 attr->rmpn = rq_obj->rmp->rmp->id;
647 attr->flush_in_error_en = 0;
648 memset(&attr->wq_attr, 0, sizeof(attr->wq_attr));
649 /* Create receive queue object with DevX. */
650 rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
652 DRV_LOG(ERR, "Can't create DevX RMP RQ object.");
657 rq_obj->rmp->ref_cnt++;
661 mlx5_devx_rq_destroy(rq_obj);
667 * Create Receive Queue using DevX API. Shared RQ is created only if rmp set.
669 * Get a pointer to partially initialized attributes structure, and updates the
678 * All other fields are updated by caller.
681 * Context returned from mlx5 open_device() glue function.
682 * @param[in/out] rq_obj
683 * Pointer to RQ to create.
684 * @param[in] wqe_size
685 * Size of WQE structure.
686 * @param[in] log_wqbb_n
687 * Log of number of WQBBs in queue.
689 * Pointer to RQ attributes structure.
691 * Socket to use for allocation.
694 * 0 on success, a negative errno value otherwise and rte_errno is set.
697 mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj,
698 uint32_t wqe_size, uint16_t log_wqbb_n,
699 struct mlx5_devx_create_rq_attr *attr, int socket)
701 if (rq_obj->rmp == NULL)
702 return mlx5_devx_rq_std_create(ctx, rq_obj, wqe_size,
703 log_wqbb_n, attr, socket);
704 return mlx5_devx_rq_shared_create(ctx, rq_obj, wqe_size,
705 log_wqbb_n, attr, socket);
709 * Change QP state to RTS.
713 * @param[in] remote_qp_id
714 * The remote QP ID for MLX5_CMD_OP_INIT2RTR_QP operation.
717 * 0 on success, a negative errno value otherwise and rte_errno is set.
720 mlx5_devx_qp2rts(struct mlx5_devx_qp *qp, uint32_t remote_qp_id)
722 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RST2INIT_QP,
724 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
728 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_INIT2RTR_QP,
730 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
734 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RTR2RTS_QP,
736 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",