vdpa/mlx5: workaround dirty bitmap MR creation

Due to kernel driver/FW issues in direct MKEY creation using the DevX
API, this patch replaces the dirty bitmap MR creation to use wrapped
mkey instead.

Fixes: 9d39e57f21ac ("vdpa/mlx5: support live migration")
Cc: stable@dpdk.org
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 62498f87fdb96f0bb3a0eeb157ea00e3f8b6d381..15212a2b30187e3e2b62cbfb29a8baae8b3c9d43 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -147,6 +147,7 @@ struct mlx5_vdpa_priv {
        struct mlx5_vdpa_steer steer;
        struct mlx5dv_var *var;
        void *virtq_db_addr;
+       struct mlx5_pmd_wrapped_mr lm_mr;
        SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list;
        struct mlx5_vdpa_virtq virtqs[];
 };

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
index 3e8d9eb9a2ec70970a74d3c89b67d56656eb90a7..e65e4faa47d980bd026fbf931236ac3c34be86e4 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
@@ -36,38 +36,22 @@ int
 mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
                           uint64_t log_size)
 {
-       struct mlx5_devx_mkey_attr mkey_attr = {
-                       .addr = (uintptr_t)log_base,
-                       .size = log_size,
-                       .pd = priv->cdev->pdn,
-                       .pg_access = 1,
-       };
        struct mlx5_devx_virtq_attr attr = {
                .type = MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS,
                .dirty_bitmap_addr = log_base,
                .dirty_bitmap_size = log_size,
        };
-       struct mlx5_vdpa_query_mr *mr = rte_malloc(__func__, sizeof(*mr), 0);
        int i;
+       int ret = mlx5_os_wrapped_mkey_create(priv->cdev->ctx, priv->cdev->pd,
+                                             priv->cdev->pdn,
+                                             (void *)(uintptr_t)log_base,
+                                             log_size, &priv->lm_mr);
 
-       if (!mr) {
-               DRV_LOG(ERR, "Failed to allocate mem for lm mr.");
+       if (!ret) {
+               DRV_LOG(ERR, "Failed to allocate wrapped MR for lm.");
                return -1;
        }
-       mr->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
-                                           (void *)(uintptr_t)log_base,
-                                           log_size, IBV_ACCESS_LOCAL_WRITE);
-       if (!mr->umem) {
-               DRV_LOG(ERR, "Failed to register umem for lm mr.");
-               goto err;
-       }
-       mkey_attr.umem_id = mr->umem->umem_id;
-       mr->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &mkey_attr);
-       if (!mr->mkey) {
-               DRV_LOG(ERR, "Failed to create Mkey for lm.");
-               goto err;
-       }
-       attr.dirty_bitmap_mkey = mr->mkey->id;
+       attr.dirty_bitmap_mkey = priv->lm_mr.lkey;
        for (i = 0; i < priv->nr_virtqs; ++i) {
                attr.queue_index = i;
                if (!priv->virtqs[i].virtq) {
@@ -78,15 +62,9 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
                        goto err;
                }
        }
-       mr->is_indirect = 0;
-       SLIST_INSERT_HEAD(&priv->mr_list, mr, next);
        return 0;
 err:
-       if (mr->mkey)
-               mlx5_devx_cmd_destroy(mr->mkey);
-       if (mr->umem)
-               mlx5_glue->devx_umem_dereg(mr->umem);
-       rte_free(mr);
+       mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
        return -1;
 }
 

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index f551a094cd2bb86ad72d8c0383c40779a2d926eb..d7707bbd91aefeb936a93f766778888fb926bc2e 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -31,6 +31,8 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
                entry = next;
        }
        SLIST_INIT(&priv->mr_list);
+       if (priv->lm_mr.addr)
+               mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
        if (priv->null_mr) {
                claim_zero(mlx5_glue->dereg_mr(priv->null_mr));
                priv->null_mr = NULL;