drivers/vdpa/mlx5/mlx5_vdpa_virtq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <string.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_errno.h>
   8
   9 #include <mlx5_common.h>
  10
  11 #include "mlx5_vdpa_utils.h"
  12 #include "mlx5_vdpa.h"
  13
  14
  15 static int
  16 mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq)
  17 {
  18         int i;
  19
  20         if (virtq->virtq) {
  21                 claim_zero(mlx5_devx_cmd_destroy(virtq->virtq));
  22                 virtq->virtq = NULL;
  23         }
  24         for (i = 0; i < 3; ++i) {
  25                 if (virtq->umems[i].obj)
  26                         claim_zero(mlx5_glue->devx_umem_dereg
  27                                                          (virtq->umems[i].obj));
  28                 if (virtq->umems[i].buf)
  29                         rte_free(virtq->umems[i].buf);
  30         }
  31         memset(&virtq->umems, 0, sizeof(virtq->umems));
  32         if (virtq->eqp.fw_qp)
  33                 mlx5_vdpa_event_qp_destroy(&virtq->eqp);
  34         return 0;
  35 }
  36
  37 void
  38 mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv)
  39 {
  40         struct mlx5_vdpa_virtq *entry;
  41         struct mlx5_vdpa_virtq *next;
  42
  43         entry = SLIST_FIRST(&priv->virtq_list);
  44         while (entry) {
  45                 next = SLIST_NEXT(entry, next);
  46                 mlx5_vdpa_virtq_unset(entry);
  47                 SLIST_REMOVE(&priv->virtq_list, entry, mlx5_vdpa_virtq, next);
  48                 rte_free(entry);
  49                 entry = next;
  50         }
  51         SLIST_INIT(&priv->virtq_list);
  52         if (priv->tis) {
  53                 claim_zero(mlx5_devx_cmd_destroy(priv->tis));
  54                 priv->tis = NULL;
  55         }
  56         if (priv->td) {
  57                 claim_zero(mlx5_devx_cmd_destroy(priv->td));
  58                 priv->td = NULL;
  59         }
  60 }
  61
  62 static uint64_t
  63 mlx5_vdpa_hva_to_gpa(struct rte_vhost_memory *mem, uint64_t hva)
  64 {
  65         struct rte_vhost_mem_region *reg;
  66         uint32_t i;
  67         uint64_t gpa = 0;
  68
  69         for (i = 0; i < mem->nregions; i++) {
  70                 reg = &mem->regions[i];
  71                 if (hva >= reg->host_user_addr &&
  72                     hva < reg->host_user_addr + reg->size) {
  73                         gpa = hva - reg->host_user_addr + reg->guest_phys_addr;
  74                         break;
  75                 }
  76         }
  77         return gpa;
  78 }
  79
  80 static int
  81 mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv,
  82                       struct mlx5_vdpa_virtq *virtq, int index)
  83 {
  84         struct rte_vhost_vring vq;
  85         struct mlx5_devx_virtq_attr attr = {0};
  86         uint64_t gpa;
  87         int ret;
  88         int i;
  89         uint16_t last_avail_idx;
  90         uint16_t last_used_idx;
  91
  92         ret = rte_vhost_get_vhost_vring(priv->vid, index, &vq);
  93         if (ret)
  94                 return -1;
  95         virtq->index = index;
  96         virtq->vq_size = vq.size;
  97         /*
  98          * No need event QPs creation when the guest in poll mode or when the
  99          * capability allows it.
 100          */
 101         attr.event_mode = vq.callfd != -1 || !(priv->caps.event_mode & (1 <<
 102                                                MLX5_VIRTQ_EVENT_MODE_NO_MSIX)) ?
 103                                                       MLX5_VIRTQ_EVENT_MODE_QP :
 104                                                   MLX5_VIRTQ_EVENT_MODE_NO_MSIX;
 105         if (attr.event_mode == MLX5_VIRTQ_EVENT_MODE_QP) {
 106                 ret = mlx5_vdpa_event_qp_create(priv, vq.size, vq.callfd,
 107                                                 &virtq->eqp);
 108                 if (ret) {
 109                         DRV_LOG(ERR, "Failed to create event QPs for virtq %d.",
 110                                 index);
 111                         return -1;
 112                 }
 113                 attr.qp_id = virtq->eqp.fw_qp->id;
 114         } else {
 115                 DRV_LOG(INFO, "Virtq %d is, for sure, working by poll mode, no"
 116                         " need event QPs and event mechanism.", index);
 117         }
 118         /* Setup 3 UMEMs for each virtq. */
 119         for (i = 0; i < 3; ++i) {
 120                 virtq->umems[i].size = priv->caps.umems[i].a * vq.size +
 121                                                           priv->caps.umems[i].b;
 122                 virtq->umems[i].buf = rte_zmalloc(__func__,
 123                                                   virtq->umems[i].size, 4096);
 124                 if (!virtq->umems[i].buf) {
 125                         DRV_LOG(ERR, "Cannot allocate umem %d memory for virtq"
 126                                 " %u.", i, index);
 127                         goto error;
 128                 }
 129                 virtq->umems[i].obj = mlx5_glue->devx_umem_reg(priv->ctx,
 130                                                         virtq->umems[i].buf,
 131                                                         virtq->umems[i].size,
 132                                                         IBV_ACCESS_LOCAL_WRITE);
 133                 if (!virtq->umems[i].obj) {
 134                         DRV_LOG(ERR, "Failed to register umem %d for virtq %u.",
 135                                 i, index);
 136                         goto error;
 137                 }
 138                 attr.umems[i].id = virtq->umems[i].obj->umem_id;
 139                 attr.umems[i].offset = 0;
 140                 attr.umems[i].size = virtq->umems[i].size;
 141         }
 142         gpa = mlx5_vdpa_hva_to_gpa(priv->vmem, (uint64_t)(uintptr_t)vq.desc);
 143         if (!gpa) {
 144                 DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
 145                 goto error;
 146         }
 147         attr.desc_addr = gpa;
 148         gpa = mlx5_vdpa_hva_to_gpa(priv->vmem, (uint64_t)(uintptr_t)vq.used);
 149         if (!gpa) {
 150                 DRV_LOG(ERR, "Fail to get GPA for used ring.");
 151                 goto error;
 152         }
 153         attr.used_addr = gpa;
 154         gpa = mlx5_vdpa_hva_to_gpa(priv->vmem, (uint64_t)(uintptr_t)vq.avail);
 155         if (!gpa) {
 156                 DRV_LOG(ERR, "Fail to get GPA for available ring.");
 157                 goto error;
 158         }
 159         attr.available_addr = gpa;
 160         rte_vhost_get_vring_base(priv->vid, index, &last_avail_idx,
 161                                  &last_used_idx);
 162         DRV_LOG(INFO, "vid %d: Init last_avail_idx=%d, last_used_idx=%d for "
 163                 "virtq %d.", priv->vid, last_avail_idx, last_used_idx, index);
 164         attr.hw_available_index = last_avail_idx;
 165         attr.hw_used_index = last_used_idx;
 166         attr.q_size = vq.size;
 167         attr.mkey = priv->gpa_mkey_index;
 168         attr.tis_id = priv->tis->id;
 169         attr.queue_index = index;
 170         virtq->virtq = mlx5_devx_cmd_create_virtq(priv->ctx, &attr);
 171         if (!virtq->virtq)
 172                 goto error;
 173         return 0;
 174 error:
 175         mlx5_vdpa_virtq_unset(virtq);
 176         return -1;
 177 }
 178
 179 int
 180 mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
 181 {
 182         struct mlx5_devx_tis_attr tis_attr = {0};
 183         struct mlx5_vdpa_virtq *virtq;
 184         uint32_t i;
 185         uint16_t nr_vring = rte_vhost_get_vring_num(priv->vid);
 186
 187         priv->td = mlx5_devx_cmd_create_td(priv->ctx);
 188         if (!priv->td) {
 189                 DRV_LOG(ERR, "Failed to create transport domain.");
 190                 return -rte_errno;
 191         }
 192         tis_attr.transport_domain = priv->td->id;
 193         priv->tis = mlx5_devx_cmd_create_tis(priv->ctx, &tis_attr);
 194         if (!priv->tis) {
 195                 DRV_LOG(ERR, "Failed to create TIS.");
 196                 goto error;
 197         }
 198         for (i = 0; i < nr_vring; i++) {
 199                 virtq = rte_zmalloc(__func__, sizeof(*virtq), 0);
 200                 if (!virtq || mlx5_vdpa_virtq_setup(priv, virtq, i)) {
 201                         if (virtq)
 202                                 rte_free(virtq);
 203                         goto error;
 204                 }
 205                 SLIST_INSERT_HEAD(&priv->virtq_list, virtq, next);
 206         }
 207         priv->nr_virtqs = nr_vring;
 208         return 0;
 209 error:
 210         mlx5_vdpa_virtqs_release(priv);
 211         return -1;
 212 }