common/mlx5: support receive memory pool

The hardware Receive Memory Pool (RMP) object holds the destination for
incoming packets/messages that are routed to the RMP through RQs. RMP
enables sharing of memory across multiple Receive Queues. Multiple
Receive Queues can be attached to the same RMP and consume memory
from that shared poll. When using RMPs, completions are reported to the
CQ pointed to by the RQ, user index that set in RQ creation time is
carried to completion entry.

This patch enables RMP based RQ, RMP is created when mlx5_devx_rq.rmp is
set.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>

diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c
index 825f84b1833d959e07eb04ea2c63fa02e7ea6dac..85b5282061abc46d363405d2a7f9d45bc6f3dd68 100644 (file)
--- a/drivers/common/mlx5/mlx5_common_devx.c
+++ b/drivers/common/mlx5/mlx5_common_devx.c
@@ -271,6 +271,39 @@ error:
        return -rte_errno;
 }
 
+/**
+ * Destroy DevX Receive Queue resources.
+ *
+ * @param[in] rq_res
+ *   DevX RQ resource to destroy.
+ */
+static void
+mlx5_devx_wq_res_destroy(struct mlx5_devx_wq_res *rq_res)
+{
+       if (rq_res->umem_obj)
+               claim_zero(mlx5_os_umem_dereg(rq_res->umem_obj));
+       if (rq_res->umem_buf)
+               mlx5_free((void *)(uintptr_t)rq_res->umem_buf);
+       memset(rq_res, 0, sizeof(*rq_res));
+}
+
+/**
+ * Destroy DevX Receive Memory Pool.
+ *
+ * @param[in] rmp
+ *   DevX RMP to destroy.
+ */
+static void
+mlx5_devx_rmp_destroy(struct mlx5_devx_rmp *rmp)
+{
+       MLX5_ASSERT(rmp->ref_cnt == 0);
+       if (rmp->rmp) {
+               claim_zero(mlx5_devx_cmd_destroy(rmp->rmp));
+               rmp->rmp = NULL;
+       }
+       mlx5_devx_wq_res_destroy(&rmp->wq);
+}
+
 /**
  * Destroy DevX Queue Pair.
  *
@@ -389,55 +422,48 @@ error:
 void
 mlx5_devx_rq_destroy(struct mlx5_devx_rq *rq)
 {
-       if (rq->rq)
+       if (rq->rq) {
                claim_zero(mlx5_devx_cmd_destroy(rq->rq));
-       if (rq->umem_obj)
-               claim_zero(mlx5_os_umem_dereg(rq->umem_obj));
-       if (rq->umem_buf)
-               mlx5_free((void *)(uintptr_t)rq->umem_buf);
+               rq->rq = NULL;
+               if (rq->rmp)
+                       rq->rmp->ref_cnt--;
+       }
+       if (rq->rmp == NULL) {
+               mlx5_devx_wq_res_destroy(&rq->wq);
+       } else {
+               if (rq->rmp->ref_cnt == 0)
+                       mlx5_devx_rmp_destroy(rq->rmp);
+       }
 }
 
 /**
- * Create Receive Queue using DevX API.
- *
- * Get a pointer to partially initialized attributes structure, and updates the
- * following fields:
- *   wq_umem_valid
- *   wq_umem_id
- *   wq_umem_offset
- *   dbr_umem_valid
- *   dbr_umem_id
- *   dbr_addr
- *   log_wq_pg_sz
- * All other fields are updated by caller.
+ * Create WQ resources using DevX API.
  *
  * @param[in] ctx
  *   Context returned from mlx5 open_device() glue function.
- * @param[in/out] rq_obj
- *   Pointer to RQ to create.
  * @param[in] wqe_size
  *   Size of WQE structure.
  * @param[in] log_wqbb_n
  *   Log of number of WQBBs in queue.
- * @param[in] attr
- *   Pointer to RQ attributes structure.
  * @param[in] socket
  *   Socket to use for allocation.
+ * @param[out] wq_attr
+ *   Pointer to WQ attributes structure.
+ * @param[out] wq_res
+ *   Pointer to WQ resource to create.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-int
-mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj, uint32_t wqe_size,
-                   uint16_t log_wqbb_n,
-                   struct mlx5_devx_create_rq_attr *attr, int socket)
+static int
+mlx5_devx_wq_init(void *ctx, uint32_t wqe_size, uint16_t log_wqbb_n, int socket,
+                 struct mlx5_devx_wq_attr *wq_attr,
+                 struct mlx5_devx_wq_res *wq_res)
 {
-       struct mlx5_devx_obj *rq = NULL;
        struct mlx5dv_devx_umem *umem_obj = NULL;
        void *umem_buf = NULL;
        size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
        uint32_t umem_size, umem_dbrec;
-       uint16_t rq_size = 1 << log_wqbb_n;
        int ret;
 
        if (alignment == (size_t)-1) {
@@ -446,7 +472,7 @@ mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj, uint32_t wqe_size,
                return -rte_errno;
        }
        /* Allocate memory buffer for WQEs and doorbell record. */
-       umem_size = wqe_size * rq_size;
+       umem_size = wqe_size * (1 << log_wqbb_n);
        umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
        umem_size += MLX5_DBR_SIZE;
        umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
@@ -464,14 +490,60 @@ mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj, uint32_t wqe_size,
                rte_errno = errno;
                goto error;
        }
+       /* Fill WQ attributes for RQ/RMP object creation. */
+       wq_attr->wq_umem_valid = 1;
+       wq_attr->wq_umem_id = mlx5_os_get_umem_id(umem_obj);
+       wq_attr->wq_umem_offset = 0;
+       wq_attr->dbr_umem_valid = 1;
+       wq_attr->dbr_umem_id = wq_attr->wq_umem_id;
+       wq_attr->dbr_addr = umem_dbrec;
+       wq_attr->log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
        /* Fill attributes for RQ object creation. */
-       attr->wq_attr.wq_umem_valid = 1;
-       attr->wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj);
-       attr->wq_attr.wq_umem_offset = 0;
-       attr->wq_attr.dbr_umem_valid = 1;
-       attr->wq_attr.dbr_umem_id = attr->wq_attr.wq_umem_id;
-       attr->wq_attr.dbr_addr = umem_dbrec;
-       attr->wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE;
+       wq_res->umem_buf = umem_buf;
+       wq_res->umem_obj = umem_obj;
+       wq_res->db_rec = RTE_PTR_ADD(umem_buf, umem_dbrec);
+       return 0;
+error:
+       ret = rte_errno;
+       if (umem_obj)
+               claim_zero(mlx5_os_umem_dereg(umem_obj));
+       if (umem_buf)
+               mlx5_free((void *)(uintptr_t)umem_buf);
+       rte_errno = ret;
+       return -rte_errno;
+}
+
+/**
+ * Create standalone Receive Queue using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in/out] rq_obj
+ *   Pointer to RQ to create.
+ * @param[in] wqe_size
+ *   Size of WQE structure.
+ * @param[in] log_wqbb_n
+ *   Log of number of WQBBs in queue.
+ * @param[in] attr
+ *   Pointer to RQ attributes structure.
+ * @param[in] socket
+ *   Socket to use for allocation.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_devx_rq_std_create(void *ctx, struct mlx5_devx_rq *rq_obj,
+                       uint32_t wqe_size, uint16_t log_wqbb_n,
+                       struct mlx5_devx_create_rq_attr *attr, int socket)
+{
+       struct mlx5_devx_obj *rq;
+       int ret;
+
+       ret = mlx5_devx_wq_init(ctx, wqe_size, log_wqbb_n, socket,
+                               &attr->wq_attr, &rq_obj->wq);
+       if (ret != 0)
+               return ret;
        /* Create receive queue object with DevX. */
        rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
        if (!rq) {
@@ -479,21 +551,160 @@ mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj, uint32_t wqe_size,
                rte_errno = ENOMEM;
                goto error;
        }
-       rq_obj->umem_buf = umem_buf;
-       rq_obj->umem_obj = umem_obj;
        rq_obj->rq = rq;
-       rq_obj->db_rec = RTE_PTR_ADD(rq_obj->umem_buf, umem_dbrec);
        return 0;
 error:
        ret = rte_errno;
-       if (umem_obj)
-               claim_zero(mlx5_os_umem_dereg(umem_obj));
-       if (umem_buf)
-               mlx5_free((void *)(uintptr_t)umem_buf);
+       mlx5_devx_wq_res_destroy(&rq_obj->wq);
+       rte_errno = ret;
+       return -rte_errno;
+}
+
+/**
+ * Create Receive Memory Pool using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in/out] rq_obj
+ *   Pointer to RQ to create.
+ * @param[in] wqe_size
+ *   Size of WQE structure.
+ * @param[in] log_wqbb_n
+ *   Log of number of WQBBs in queue.
+ * @param[in] attr
+ *   Pointer to RQ attributes structure.
+ * @param[in] socket
+ *   Socket to use for allocation.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_devx_rmp_create(void *ctx, struct mlx5_devx_rmp *rmp_obj,
+                    uint32_t wqe_size, uint16_t log_wqbb_n,
+                    struct mlx5_devx_wq_attr *wq_attr, int socket)
+{
+       struct mlx5_devx_create_rmp_attr rmp_attr = { 0 };
+       int ret;
+
+       if (rmp_obj->rmp != NULL)
+               return 0;
+       rmp_attr.wq_attr = *wq_attr;
+       ret = mlx5_devx_wq_init(ctx, wqe_size, log_wqbb_n, socket,
+                               &rmp_attr.wq_attr, &rmp_obj->wq);
+       if (ret != 0)
+               return ret;
+       rmp_attr.state = MLX5_RMPC_STATE_RDY;
+       rmp_attr.basic_cyclic_rcv_wqe =
+               wq_attr->wq_type != MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ;
+       /* Create receive memory pool object with DevX. */
+       rmp_obj->rmp = mlx5_devx_cmd_create_rmp(ctx, &rmp_attr, socket);
+       if (rmp_obj->rmp == NULL) {
+               DRV_LOG(ERR, "Can't create DevX RMP object.");
+               rte_errno = ENOMEM;
+               goto error;
+       }
+       return 0;
+error:
+       ret = rte_errno;
+       mlx5_devx_wq_res_destroy(&rmp_obj->wq);
+       rte_errno = ret;
+       return -rte_errno;
+}
+
+/**
+ * Create Shared Receive Queue based on RMP using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in/out] rq_obj
+ *   Pointer to RQ to create.
+ * @param[in] wqe_size
+ *   Size of WQE structure.
+ * @param[in] log_wqbb_n
+ *   Log of number of WQBBs in queue.
+ * @param[in] attr
+ *   Pointer to RQ attributes structure.
+ * @param[in] socket
+ *   Socket to use for allocation.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_devx_rq_shared_create(void *ctx, struct mlx5_devx_rq *rq_obj,
+                          uint32_t wqe_size, uint16_t log_wqbb_n,
+                          struct mlx5_devx_create_rq_attr *attr, int socket)
+{
+       struct mlx5_devx_obj *rq;
+       int ret;
+
+       ret = mlx5_devx_rmp_create(ctx, rq_obj->rmp, wqe_size, log_wqbb_n,
+                                  &attr->wq_attr, socket);
+       if (ret != 0)
+               return ret;
+       attr->mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_RMP;
+       attr->rmpn = rq_obj->rmp->rmp->id;
+       attr->flush_in_error_en = 0;
+       memset(&attr->wq_attr, 0, sizeof(attr->wq_attr));
+       /* Create receive queue object with DevX. */
+       rq = mlx5_devx_cmd_create_rq(ctx, attr, socket);
+       if (!rq) {
+               DRV_LOG(ERR, "Can't create DevX RMP RQ object.");
+               rte_errno = ENOMEM;
+               goto error;
+       }
+       rq_obj->rq = rq;
+       rq_obj->rmp->ref_cnt++;
+       return 0;
+error:
+       ret = rte_errno;
+       mlx5_devx_rq_destroy(rq_obj);
        rte_errno = ret;
        return -rte_errno;
 }
 
+/**
+ * Create Receive Queue using DevX API. Shared RQ is created only if rmp set.
+ *
+ * Get a pointer to partially initialized attributes structure, and updates the
+ * following fields:
+ *   wq_umem_valid
+ *   wq_umem_id
+ *   wq_umem_offset
+ *   dbr_umem_valid
+ *   dbr_umem_id
+ *   dbr_addr
+ *   log_wq_pg_sz
+ * All other fields are updated by caller.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in/out] rq_obj
+ *   Pointer to RQ to create.
+ * @param[in] wqe_size
+ *   Size of WQE structure.
+ * @param[in] log_wqbb_n
+ *   Log of number of WQBBs in queue.
+ * @param[in] attr
+ *   Pointer to RQ attributes structure.
+ * @param[in] socket
+ *   Socket to use for allocation.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_devx_rq_create(void *ctx, struct mlx5_devx_rq *rq_obj,
+                   uint32_t wqe_size, uint16_t log_wqbb_n,
+                   struct mlx5_devx_create_rq_attr *attr, int socket)
+{
+       if (rq_obj->rmp == NULL)
+               return mlx5_devx_rq_std_create(ctx, rq_obj, wqe_size,
+                                              log_wqbb_n, attr, socket);
+       return mlx5_devx_rq_shared_create(ctx, rq_obj, wqe_size,
+                                         log_wqbb_n, attr, socket);
+}
 
 /**
  * Change QP state to RTS.

diff --git a/drivers/common/mlx5/mlx5_common_devx.h b/drivers/common/mlx5/mlx5_common_devx.h
index f699405f69b46fdfc2fe823bc0cf1cf8cde945bb..7ceac040f8b0b66155cfdb2408827bbfda08e002 100644 (file)
--- a/drivers/common/mlx5/mlx5_common_devx.h
+++ b/drivers/common/mlx5/mlx5_common_devx.h
@@ -45,14 +45,27 @@ struct mlx5_devx_qp {
        volatile uint32_t *db_rec; /* The QP doorbell record. */
 };
 
-/* DevX Receive Queue structure. */
-struct mlx5_devx_rq {
-       struct mlx5_devx_obj *rq; /* The RQ DevX object. */
+/* DevX Receive Queue resource structure. */
+struct mlx5_devx_wq_res {
        void *umem_obj; /* The RQ umem object. */
        volatile void *umem_buf;
        volatile uint32_t *db_rec; /* The RQ doorbell record. */
 };
 
+/* DevX Receive Memory Pool structure. */
+struct mlx5_devx_rmp {
+       struct mlx5_devx_obj *rmp; /* The RMP DevX object. */
+       uint32_t ref_cnt; /* Reference count. */
+       struct mlx5_devx_wq_res wq;
+};
+
+/* DevX Receive Queue structure. */
+struct mlx5_devx_rq {
+       struct mlx5_devx_obj *rq; /* The RQ DevX object. */
+       struct mlx5_devx_rmp *rmp; /* Shared RQ RMP object. */
+       struct mlx5_devx_wq_res wq; /* WQ resource of standalone RQ. */
+};
+
 /* mlx5_common_devx.c */
 
 __rte_internal

diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
index 424f77be79036210f6373b1ce07e5cea3438fee3..443252df05d7508be2bbd359fc34c680479c3fe2 100644 (file)
--- a/drivers/net/mlx5/mlx5_devx.c
+++ b/drivers/net/mlx5/mlx5_devx.c
@@ -515,8 +515,8 @@ mlx5_rxq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx)
        ret = mlx5_devx_modify_rq(tmpl, MLX5_RXQ_MOD_RST2RDY);
        if (ret)
                goto error;
-       rxq_data->wqes = (void *)(uintptr_t)tmpl->rq_obj.umem_buf;
-       rxq_data->rq_db = (uint32_t *)(uintptr_t)tmpl->rq_obj.db_rec;
+       rxq_data->wqes = (void *)(uintptr_t)tmpl->rq_obj.wq.umem_buf;
+       rxq_data->rq_db = (uint32_t *)(uintptr_t)tmpl->rq_obj.wq.db_rec;
        rxq_data->cq_arm_sn = 0;
        rxq_data->cq_ci = 0;
        mlx5_rxq_initialize(rxq_data);