vdpa/mlx5: workaround guest MR registrations

Due to kernel issue in direct MKEY creation using the DevX API, this
patch replaces the virtio MR creation to use Verbs API.

Fixes: cc07a42da250 ("vdpa/mlx5: prepare memory regions")
Cc: stable@dpdk.org
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 15212a2..2261792 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -59,10 +59,10 @@ struct mlx5_vdpa_event_qp {
 
 struct mlx5_vdpa_query_mr {
        SLIST_ENTRY(mlx5_vdpa_query_mr) next;
-       void *addr;
-       uint64_t length;
-       struct mlx5dv_devx_umem *umem;
-       struct mlx5_devx_obj *mkey;
+       union {
+               struct ibv_mr *mr;
+               struct mlx5_devx_obj *mkey;
+       };
        int is_indirect;
 };
 

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index d7707bb..b1b9053 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -23,9 +23,10 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
        entry = SLIST_FIRST(&priv->mr_list);
        while (entry) {
                next = SLIST_NEXT(entry, next);
-               claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
-               if (!entry->is_indirect)
-                       claim_zero(mlx5_glue->devx_umem_dereg(entry->umem));
+               if (entry->is_indirect)
+                       claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
+               else
+                       claim_zero(mlx5_glue->dereg_mr(entry->mr));
                SLIST_REMOVE(&priv->mr_list, entry, mlx5_vdpa_query_mr, next);
                rte_free(entry);
                entry = next;
@@ -202,7 +203,6 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
                goto error;
        }
        DRV_LOG(DEBUG, "Dump fill Mkey = %u.", priv->null_mr->lkey);
-       memset(&mkey_attr, 0, sizeof(mkey_attr));
        for (i = 0; i < mem->nregions; i++) {
                reg = &mem->regions[i];
                entry = rte_zmalloc(__func__, sizeof(*entry), 0);
@@ -211,28 +211,15 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
                        DRV_LOG(ERR, "Failed to allocate mem entry memory.");
                        goto error;
                }
-               entry->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
-                                        (void *)(uintptr_t)reg->host_user_addr,
-                                            reg->size, IBV_ACCESS_LOCAL_WRITE);
-               if (!entry->umem) {
-                       DRV_LOG(ERR, "Failed to register Umem by Devx.");
-                       ret = -errno;
-                       goto error;
-               }
-               mkey_attr.addr = (uintptr_t)(reg->guest_phys_addr);
-               mkey_attr.size = reg->size;
-               mkey_attr.umem_id = entry->umem->umem_id;
-               mkey_attr.pd = priv->cdev->pdn;
-               mkey_attr.pg_access = 1;
-               entry->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx,
-                                                       &mkey_attr);
-               if (!entry->mkey) {
+               entry->mr = mlx5_glue->reg_mr_iova(priv->cdev->pd,
+                                      (void *)(uintptr_t)(reg->host_user_addr),
+                                      reg->size, reg->guest_phys_addr,
+                                      IBV_ACCESS_LOCAL_WRITE);
+               if (!entry->mr) {
                        DRV_LOG(ERR, "Failed to create direct Mkey.");
                        ret = -rte_errno;
                        goto error;
                }
-               entry->addr = (void *)(uintptr_t)(reg->host_user_addr);
-               entry->length = reg->size;
                entry->is_indirect = 0;
                if (i > 0) {
                        uint64_t sadd;
@@ -262,12 +249,13 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
                for (k = 0; k < reg->size; k += klm_size) {
                        klm_array[klm_index].byte_count = k + klm_size >
                                           reg->size ? reg->size - k : klm_size;
-                       klm_array[klm_index].mkey = entry->mkey->id;
+                       klm_array[klm_index].mkey = entry->mr->lkey;
                        klm_array[klm_index].address = reg->guest_phys_addr + k;
                        klm_index++;
                }
                SLIST_INSERT_HEAD(&priv->mr_list, entry, next);
        }
+       memset(&mkey_attr, 0, sizeof(mkey_attr));
        mkey_attr.addr = (uintptr_t)(mem->regions[0].guest_phys_addr);
        mkey_attr.size = mem_size;
        mkey_attr.pd = priv->cdev->pdn;
@@ -295,13 +283,8 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
        priv->gpa_mkey_index = entry->mkey->id;
        return 0;
 error:
-       if (entry) {
-               if (entry->mkey)
-                       mlx5_devx_cmd_destroy(entry->mkey);
-               if (entry->umem)
-                       mlx5_glue->devx_umem_dereg(entry->umem);
+       if (entry)
                rte_free(entry);
-       }
        mlx5_vdpa_mem_dereg(priv);
        rte_errno = -ret;
        return ret;