1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
7 #include <rte_common.h>
8 #include <rte_eal_paging.h>
10 #include <mlx5_glue.h>
11 #include <mlx5_common_os.h>
14 #include "mlx5_devx_cmds.h"
15 #include "mlx5_common_log.h"
16 #include "mlx5_malloc.h"
17 #include "mlx5_common.h"
18 #include "mlx5_common_devx.h"
21 * Destroy DevX Completion Queue.
27 mlx5_devx_cq_destroy(struct mlx5_devx_cq *cq)
30 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
32 claim_zero(mlx5_os_umem_dereg(cq->umem_obj));
34 mlx5_free((void *)(uintptr_t)cq->umem_buf);
37 /* Mark all CQEs initially as invalid. */
39 mlx5_cq_init(struct mlx5_devx_cq *cq_obj, uint16_t cq_size)
41 volatile struct mlx5_cqe *cqe = cq_obj->cqes;
44 for (i = 0; i < cq_size; i++, cqe++)
45 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK;
49 * Create Completion Queue using DevX API.
51 * Get a pointer to partially initialized attributes structure, and updates the
62 * All other fields are updated by caller.
65 * Context returned from mlx5 open_device() glue function.
66 * @param[in/out] cq_obj
67 * Pointer to CQ to create.
68 * @param[in] log_desc_n
69 * Log of number of descriptors in queue.
71 * Pointer to CQ attributes structure.
73 * Socket to use for allocation.
76 * 0 on success, a negative errno value otherwise and rte_errno is set.
79 mlx5_devx_cq_create(void *ctx, struct mlx5_devx_cq *cq_obj, uint16_t log_desc_n,
80 struct mlx5_devx_cq_attr *attr, int socket)
82 struct mlx5_devx_obj *cq = NULL;
83 struct mlx5dv_devx_umem *umem_obj = NULL;
84 void *umem_buf = NULL;
85 size_t page_size = rte_mem_page_size();
86 size_t alignment = MLX5_CQE_BUF_ALIGNMENT;
87 uint32_t umem_size, umem_dbrec;
89 uint32_t num_of_cqes = RTE_BIT32(log_desc_n);
92 if (page_size == (size_t)-1 || alignment == (size_t)-1) {
93 DRV_LOG(ERR, "Failed to get page_size.");
97 /* Query first EQN. */
98 ret = mlx5_glue->devx_query_eqn(ctx, 0, &eqn);
101 DRV_LOG(ERR, "Failed to query event queue number.");
104 /* Allocate memory buffer for CQEs and doorbell record. */
105 umem_size = sizeof(struct mlx5_cqe) * num_of_cqes;
106 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
107 umem_size += MLX5_DBR_SIZE;
108 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
111 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
115 /* Register allocated buffer in user space with DevX. */
116 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
117 IBV_ACCESS_LOCAL_WRITE);
119 DRV_LOG(ERR, "Failed to register umem for CQ.");
123 /* Fill attributes for CQ object creation. */
124 attr->q_umem_valid = 1;
125 attr->q_umem_id = mlx5_os_get_umem_id(umem_obj);
126 attr->q_umem_offset = 0;
127 attr->db_umem_valid = 1;
128 attr->db_umem_id = attr->q_umem_id;
129 attr->db_umem_offset = umem_dbrec;
131 attr->log_cq_size = log_desc_n;
132 attr->log_page_size = rte_log2_u32(page_size);
133 /* Create completion queue object with DevX. */
134 cq = mlx5_devx_cmd_create_cq(ctx, attr);
136 DRV_LOG(ERR, "Can't create DevX CQ object.");
140 cq_obj->umem_buf = umem_buf;
141 cq_obj->umem_obj = umem_obj;
143 cq_obj->db_rec = RTE_PTR_ADD(cq_obj->umem_buf, umem_dbrec);
144 /* Mark all CQEs initially as invalid. */
145 mlx5_cq_init(cq_obj, num_of_cqes);
150 claim_zero(mlx5_os_umem_dereg(umem_obj));
152 mlx5_free((void *)(uintptr_t)umem_buf);
158 * Destroy DevX Send Queue.
161 * DevX SQ to destroy.
164 mlx5_devx_sq_destroy(struct mlx5_devx_sq *sq)
167 claim_zero(mlx5_devx_cmd_destroy(sq->sq));
169 claim_zero(mlx5_os_umem_dereg(sq->umem_obj));
171 mlx5_free((void *)(uintptr_t)sq->umem_buf);
175 * Create Send Queue using DevX API.
177 * Get a pointer to partially initialized attributes structure, and updates the
189 * All other fields are updated by caller.
192 * Context returned from mlx5 open_device() glue function.
193 * @param[in/out] sq_obj
194 * Pointer to SQ to create.
195 * @param[in] log_wqbb_n
196 * Log of number of WQBBs in queue.
198 * Pointer to SQ attributes structure.
200 * Socket to use for allocation.
203 * 0 on success, a negative errno value otherwise and rte_errno is set.
206 mlx5_devx_sq_create(void *ctx, struct mlx5_devx_sq *sq_obj, uint16_t log_wqbb_n,
207 struct mlx5_devx_create_sq_attr *attr, int socket)
209 struct mlx5_devx_obj *sq = NULL;
210 struct mlx5dv_devx_umem *umem_obj = NULL;
211 void *umem_buf = NULL;
212 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
213 uint32_t umem_size, umem_dbrec;
214 uint32_t num_of_wqbbs = RTE_BIT32(log_wqbb_n);
217 if (alignment == (size_t)-1) {
218 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
222 /* Allocate memory buffer for WQEs and doorbell record. */
223 umem_size = MLX5_WQE_SIZE * num_of_wqbbs;
224 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
225 umem_size += MLX5_DBR_SIZE;
226 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
229 DRV_LOG(ERR, "Failed to allocate memory for SQ.");
233 /* Register allocated buffer in user space with DevX. */
234 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
235 IBV_ACCESS_LOCAL_WRITE);
237 DRV_LOG(ERR, "Failed to register umem for SQ.");
241 /* Fill attributes for SQ object creation. */
242 attr->wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;
243 attr->wq_attr.wq_umem_valid = 1;
244 attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj);
245 attr->wq_attr.wq_umem_offset = 0;
246 attr->wq_attr.dbr_umem_valid = 1;
247 attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id;
248 attr->wq_attr.dbr_addr = umem_dbrec;
249 attr->wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE);
250 attr->wq_attr.log_wq_sz = log_wqbb_n;
251 attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
252 /* Create send queue object with DevX. */
253 sq = mlx5_devx_cmd_create_sq(ctx, attr);
255 DRV_LOG(ERR, "Can't create DevX SQ object.");
259 sq_obj->umem_buf = umem_buf;
260 sq_obj->umem_obj = umem_obj;
262 sq_obj->db_rec = RTE_PTR_ADD(sq_obj->umem_buf, umem_dbrec);
267 claim_zero(mlx5_os_umem_dereg(umem_obj));
269 mlx5_free((void *)(uintptr_t)umem_buf);
275 * Destroy DevX Receive Queue resources.
278 * DevX RQ resource to destroy.
281 mlx5_devx_wq_res_destroy(struct mlx5_devx_wq_res *rq_res)
283 if (rq_res->umem_obj)
284 claim_zero(mlx5_os_umem_dereg(rq_res->umem_obj));
285 if (rq_res->umem_buf)
286 mlx5_free((void *)(uintptr_t)rq_res->umem_buf);
287 memset(rq_res, 0, sizeof(*rq_res));
291 * Destroy DevX Receive Memory Pool.
294 * DevX RMP to destroy.
297 mlx5_devx_rmp_destroy(struct mlx5_devx_rmp *rmp)
299 MLX5_ASSERT(rmp->ref_cnt == 0);
301 claim_zero(mlx5_devx_cmd_destroy(rmp->rmp));
304 mlx5_devx_wq_res_destroy(&rmp->wq);
308 * Destroy DevX Queue Pair.
311 * DevX QP to destroy.
314 mlx5_devx_qp_destroy(struct mlx5_devx_qp *qp)
317 claim_zero(mlx5_devx_cmd_destroy(qp->qp));
319 claim_zero(mlx5_os_umem_dereg(qp->umem_obj));
321 mlx5_free((void *)(uintptr_t)qp->umem_buf);
325 * Create Queue Pair using DevX API.
327 * Get a pointer to partially initialized attributes structure, and updates the
335 * All other fields are updated by caller.
338 * Context returned from mlx5 open_device() glue function.
339 * @param[in/out] qp_obj
340 * Pointer to QP to create.
341 * @param[in] log_wqbb_n
342 * Log of number of WQBBs in queue.
344 * Pointer to QP attributes structure.
346 * Socket to use for allocation.
349 * 0 on success, a negative errno value otherwise and rte_errno is set.
352 mlx5_devx_qp_create(void *ctx, struct mlx5_devx_qp *qp_obj, uint32_t log_wqbb_n,
353 struct mlx5_devx_qp_attr *attr, int socket)
355 struct mlx5_devx_obj *qp = NULL;
356 struct mlx5dv_devx_umem *umem_obj = NULL;
357 void *umem_buf = NULL;
358 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
359 uint32_t umem_size, umem_dbrec;
360 uint32_t num_of_wqbbs = RTE_BIT32(log_wqbb_n);
363 if (alignment == (size_t)-1) {
364 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
368 /* Allocate memory buffer for WQEs and doorbell record. */
369 umem_size = MLX5_WQE_SIZE * num_of_wqbbs;
370 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
371 umem_size += MLX5_DBR_SIZE;
372 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
375 DRV_LOG(ERR, "Failed to allocate memory for QP.");
379 /* Register allocated buffer in user space with DevX. */
380 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf, umem_size,
381 IBV_ACCESS_LOCAL_WRITE);
383 DRV_LOG(ERR, "Failed to register umem for QP.");
387 /* Fill attributes for SQ object creation. */
388 attr->wq_umem_id = mlx5_os_get_umem_id(umem_obj);
389 attr->wq_umem_offset = 0;
390 attr->dbr_umem_valid = 1;
391 attr->dbr_umem_id = attr->wq_umem_id;
392 attr->dbr_address = umem_dbrec;
393 attr->log_page_size = MLX5_LOG_PAGE_SIZE;
394 /* Create send queue object with DevX. */
395 qp = mlx5_devx_cmd_create_qp(ctx, attr);
397 DRV_LOG(ERR, "Can't create DevX QP object.");
401 qp_obj->umem_buf = umem_buf;
402 qp_obj->umem_obj = umem_obj;
404 qp_obj->db_rec = RTE_PTR_ADD(qp_obj->umem_buf, umem_dbrec);
409 claim_zero(mlx5_os_umem_dereg(umem_obj));
411 mlx5_free((void *)(uintptr_t)umem_buf);
417 * Destroy DevX Receive Queue.
420 * DevX RQ to destroy.
423 mlx5_devx_rq_destroy(struct mlx5_devx_rq *rq)
426 claim_zero(mlx5_devx_cmd_destroy(rq->rq));
431 if (rq->rmp == NULL) {
432 mlx5_devx_wq_res_destroy(&rq->wq);
434 if (rq->rmp->ref_cnt == 0)
435 mlx5_devx_rmp_destroy(rq->rmp);
440 * Create WQ resources using DevX API.
443 * Context returned from mlx5 open_device() glue function.
444 * @param[in] wqe_size
445 * Size of WQE structure.
446 * @param[in] log_wqbb_n
447 * Log of number of WQBBs in queue.
449 * Socket to use for allocation.
450 * @param[out] wq_attr
451 * Pointer to WQ attributes structure.
453 * Pointer to WQ resource to create.
456 * 0 on success, a negative errno value otherwise and rte_errno is set.
459 mlx5_devx_wq_init(void *ctx, uint32_t wqe_size, uint16_t log_wqbb_n, int socket,
460 struct mlx5_devx_wq_attr *wq_attr,
461 struct mlx5_devx_wq_res *wq_res)
463 struct mlx5dv_devx_umem *umem_obj = NULL;
464 void *umem_buf = NULL;
465 size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
466 uint32_t umem_size, umem_dbrec;
469 if (alignment == (size_t)-1) {
470 DRV_LOG(ERR, "Failed to get WQE buf alignment.");
474 /* Allocate memory buffer for WQEs and doorbell record. */
475 umem_size = wqe_size * (1 << log_wqbb_n);
476 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
477 umem_size += MLX5_DBR_SIZE;
478 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
481 DRV_LOG(ERR, "Failed to allocate memory for RQ.");
485 /* Register allocated buffer in user space with DevX. */
486 umem_obj = mlx5_os_umem_reg(ctx, (void *)(uintptr_t)umem_buf,
489 DRV_LOG(ERR, "Failed to register umem for RQ.");
493 /* Fill WQ attributes for RQ/RMP object creation. */
494 wq_attr->wq_umem_valid = 1;
495 wq_attr->wq_umem_id = mlx5_os_get_umem_id(umem_obj);
496 wq_attr->wq_umem_offset = 0;
497 wq_attr->dbr_umem_valid = 1;
498 wq_attr->dbr_umem_id = wq_attr->wq_umem_id;
499 wq_attr->dbr_addr = umem_dbrec;
500 wq_attr->log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
501 /* Fill attributes for RQ object creation. */
502 wq_res->umem_buf = umem_buf;
503 wq_res->umem_obj = umem_obj;
504 wq_res->db_rec = RTE_PTR_ADD(umem_buf, umem_dbrec);
509 claim_zero(mlx5_os_umem_dereg(umem_obj));
511 mlx5_free((void *)(uintptr_t)umem_buf);
517 * Create standalone Receive Queue using DevX API.
520 * Context returned from mlx5 open_device() glue function.
521 * @param[in/out] rq_obj
522 * Pointer to RQ to create.
523 * @param[in] wqe_size
524 * Size of WQE structure.
525 * @param[in] log_wqbb_n
526 * Log of number of WQBBs in queue.
528 * Pointer to RQ attributes structure.
530 * Socket to use for allocation.
533 * 0 on success, a negative errno value otherwise and rte_errno is set.
536 mlx5_devx_rq_std_create(void *ctx, struct mlx5_devx_rq *rq_obj,
537 uint32_t wqe_size, uint16_t log_wqbb_n,
538 struct mlx5_devx_create_rq_attr *attr, int socket)
540 struct mlx5_devx_obj *rq;
543 ret = mlx5_devx_wq_init(ctx, wqe_size, log_wqbb_n, socket,
544 &attr->wq_attr, &rq_obj->wq);
547 /* Create receive queue object with DevX. */
548 rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
550 DRV_LOG(ERR, "Can't create DevX RQ object.");
558 mlx5_devx_wq_res_destroy(&rq_obj->wq);
564 * Create Receive Memory Pool using DevX API.
567 * Context returned from mlx5 open_device() glue function.
568 * @param[in/out] rq_obj
569 * Pointer to RQ to create.
570 * @param[in] wqe_size
571 * Size of WQE structure.
572 * @param[in] log_wqbb_n
573 * Log of number of WQBBs in queue.
575 * Pointer to RQ attributes structure.
577 * Socket to use for allocation.
580 * 0 on success, a negative errno value otherwise and rte_errno is set.
583 mlx5_devx_rmp_create(void *ctx, struct mlx5_devx_rmp *rmp_obj,
584 uint32_t wqe_size, uint16_t log_wqbb_n,
585 struct mlx5_devx_wq_attr *wq_attr, int socket)
587 struct mlx5_devx_create_rmp_attr rmp_attr = { 0 };
590 if (rmp_obj->rmp != NULL)
592 rmp_attr.wq_attr = *wq_attr;
593 ret = mlx5_devx_wq_init(ctx, wqe_size, log_wqbb_n, socket,
594 &rmp_attr.wq_attr, &rmp_obj->wq);
597 rmp_attr.state = MLX5_RMPC_STATE_RDY;
598 rmp_attr.basic_cyclic_rcv_wqe =
599 wq_attr->wq_type != MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ;
600 /* Create receive memory pool object with DevX. */
601 rmp_obj->rmp = mlx5_devx_cmd_create_rmp(ctx, &rmp_attr, socket);
602 if (rmp_obj->rmp == NULL) {
603 DRV_LOG(ERR, "Can't create DevX RMP object.");
610 mlx5_devx_wq_res_destroy(&rmp_obj->wq);
616 * Create Shared Receive Queue based on RMP using DevX API.
619 * Context returned from mlx5 open_device() glue function.
620 * @param[in/out] rq_obj
621 * Pointer to RQ to create.
622 * @param[in] wqe_size
623 * Size of WQE structure.
624 * @param[in] log_wqbb_n
625 * Log of number of WQBBs in queue.
627 * Pointer to RQ attributes structure.
629 * Socket to use for allocation.
632 * 0 on success, a negative errno value otherwise and rte_errno is set.
635 mlx5_devx_rq_shared_create(void *ctx, struct mlx5_devx_rq *rq_obj,
636 uint32_t wqe_size, uint16_t log_wqbb_n,
637 struct mlx5_devx_create_rq_attr *attr, int socket)
639 struct mlx5_devx_obj *rq;
642 ret = mlx5_devx_rmp_create(ctx, rq_obj->rmp, wqe_size, log_wqbb_n,
643 &attr->wq_attr, socket);
646 attr->mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_RMP;
647 attr->rmpn = rq_obj->rmp->rmp->id;
648 attr->flush_in_error_en = 0;
649 memset(&attr->wq_attr, 0, sizeof(attr->wq_attr));
650 /* Create receive queue object with DevX. */
651 rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
653 DRV_LOG(ERR, "Can't create DevX RMP RQ object.");
658 rq_obj->rmp->ref_cnt++;
662 mlx5_devx_rq_destroy(rq_obj);
668 * Create Receive Queue using DevX API. Shared RQ is created only if rmp set.
670 * Get a pointer to partially initialized attributes structure, and updates the
679 * All other fields are updated by caller.
682 * Context returned from mlx5 open_device() glue function.
683 * @param[in/out] rq_obj
684 * Pointer to RQ to create.
685 * @param[in] wqe_size
686 * Size of WQE structure.
687 * @param[in] log_wqbb_n
688 * Log of number of WQBBs in queue.
690 * Pointer to RQ attributes structure.
692 * Socket to use for allocation.
695 * 0 on success, a negative errno value otherwise and rte_errno is set.
698 mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj,
699 uint32_t wqe_size, uint16_t log_wqbb_n,
700 struct mlx5_devx_create_rq_attr *attr, int socket)
702 if (rq_obj->rmp == NULL)
703 return mlx5_devx_rq_std_create(ctx, rq_obj, wqe_size,
704 log_wqbb_n, attr, socket);
705 return mlx5_devx_rq_shared_create(ctx, rq_obj, wqe_size,
706 log_wqbb_n, attr, socket);
710 * Change QP state to RTS.
714 * @param[in] remote_qp_id
715 * The remote QP ID for MLX5_CMD_OP_INIT2RTR_QP operation.
718 * 0 on success, a negative errno value otherwise and rte_errno is set.
721 mlx5_devx_qp2rts(struct mlx5_devx_qp *qp, uint32_t remote_qp_id)
723 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RST2INIT_QP,
725 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
729 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_INIT2RTR_QP,
731 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
735 if (mlx5_devx_cmd_modify_qp_state(qp->qp, MLX5_CMD_OP_RTR2RTS_QP,
737 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",