drivers/vdpa/mlx5/mlx5_vdpa_virtq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <string.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_errno.h>
   8
   9 #include <mlx5_common.h>
  10
  11 #include "mlx5_vdpa_utils.h"
  12 #include "mlx5_vdpa.h"
  13
  14
  15 static int
  16 mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq)
  17 {
  18         int i;
  19
  20         if (virtq->virtq) {
  21                 claim_zero(mlx5_devx_cmd_destroy(virtq->virtq));
  22                 virtq->virtq = NULL;
  23         }
  24         for (i = 0; i < 3; ++i) {
  25                 if (virtq->umems[i].obj)
  26                         claim_zero(mlx5_glue->devx_umem_dereg
  27                                                          (virtq->umems[i].obj));
  28                 if (virtq->umems[i].buf)
  29                         rte_free(virtq->umems[i].buf);
  30         }
  31         memset(&virtq->umems, 0, sizeof(virtq->umems));
  32         if (virtq->eqp.fw_qp)
  33                 mlx5_vdpa_event_qp_destroy(&virtq->eqp);
  34         return 0;
  35 }
  36
  37 void
  38 mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv)
  39 {
  40         struct mlx5_vdpa_virtq *entry;
  41         struct mlx5_vdpa_virtq *next;
  42
  43         entry = SLIST_FIRST(&priv->virtq_list);
  44         while (entry) {
  45                 next = SLIST_NEXT(entry, next);
  46                 mlx5_vdpa_virtq_unset(entry);
  47                 SLIST_REMOVE(&priv->virtq_list, entry, mlx5_vdpa_virtq, next);
  48                 rte_free(entry);
  49                 entry = next;
  50         }
  51         SLIST_INIT(&priv->virtq_list);
  52         if (priv->tis) {
  53                 claim_zero(mlx5_devx_cmd_destroy(priv->tis));
  54                 priv->tis = NULL;
  55         }
  56         if (priv->td) {
  57                 claim_zero(mlx5_devx_cmd_destroy(priv->td));
  58                 priv->td = NULL;
  59         }
  60         priv->features = 0;
  61 }
  62
  63 static uint64_t
  64 mlx5_vdpa_hva_to_gpa(struct rte_vhost_memory *mem, uint64_t hva)
  65 {
  66         struct rte_vhost_mem_region *reg;
  67         uint32_t i;
  68         uint64_t gpa = 0;
  69
  70         for (i = 0; i < mem->nregions; i++) {
  71                 reg = &mem->regions[i];
  72                 if (hva >= reg->host_user_addr &&
  73                     hva < reg->host_user_addr + reg->size) {
  74                         gpa = hva - reg->host_user_addr + reg->guest_phys_addr;
  75                         break;
  76                 }
  77         }
  78         return gpa;
  79 }
  80
  81 static int
  82 mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv,
  83                       struct mlx5_vdpa_virtq *virtq, int index)
  84 {
  85         struct rte_vhost_vring vq;
  86         struct mlx5_devx_virtq_attr attr = {0};
  87         uint64_t gpa;
  88         int ret;
  89         int i;
  90         uint16_t last_avail_idx;
  91         uint16_t last_used_idx;
  92
  93         ret = rte_vhost_get_vhost_vring(priv->vid, index, &vq);
  94         if (ret)
  95                 return -1;
  96         virtq->index = index;
  97         virtq->vq_size = vq.size;
  98         attr.tso_ipv4 = !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO4));
  99         attr.tso_ipv6 = !!(priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO6));
 100         attr.tx_csum = !!(priv->features & (1ULL << VIRTIO_NET_F_CSUM));
 101         attr.rx_csum = !!(priv->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM));
 102         attr.virtio_version_1_0 = !!(priv->features & (1ULL <<
 103                                                         VIRTIO_F_VERSION_1));
 104         attr.type = (priv->features & (1ULL << VIRTIO_F_RING_PACKED)) ?
 105                         MLX5_VIRTQ_TYPE_PACKED : MLX5_VIRTQ_TYPE_SPLIT;
 106         /*
 107          * No need event QPs creation when the guest in poll mode or when the
 108          * capability allows it.
 109          */
 110         attr.event_mode = vq.callfd != -1 || !(priv->caps.event_mode & (1 <<
 111                                                MLX5_VIRTQ_EVENT_MODE_NO_MSIX)) ?
 112                                                       MLX5_VIRTQ_EVENT_MODE_QP :
 113                                                   MLX5_VIRTQ_EVENT_MODE_NO_MSIX;
 114         if (attr.event_mode == MLX5_VIRTQ_EVENT_MODE_QP) {
 115                 ret = mlx5_vdpa_event_qp_create(priv, vq.size, vq.callfd,
 116                                                 &virtq->eqp);
 117                 if (ret) {
 118                         DRV_LOG(ERR, "Failed to create event QPs for virtq %d.",
 119                                 index);
 120                         return -1;
 121                 }
 122                 attr.qp_id = virtq->eqp.fw_qp->id;
 123         } else {
 124                 DRV_LOG(INFO, "Virtq %d is, for sure, working by poll mode, no"
 125                         " need event QPs and event mechanism.", index);
 126         }
 127         /* Setup 3 UMEMs for each virtq. */
 128         for (i = 0; i < 3; ++i) {
 129                 virtq->umems[i].size = priv->caps.umems[i].a * vq.size +
 130                                                           priv->caps.umems[i].b;
 131                 virtq->umems[i].buf = rte_zmalloc(__func__,
 132                                                   virtq->umems[i].size, 4096);
 133                 if (!virtq->umems[i].buf) {
 134                         DRV_LOG(ERR, "Cannot allocate umem %d memory for virtq"
 135                                 " %u.", i, index);
 136                         goto error;
 137                 }
 138                 virtq->umems[i].obj = mlx5_glue->devx_umem_reg(priv->ctx,
 139                                                         virtq->umems[i].buf,
 140                                                         virtq->umems[i].size,
 141                                                         IBV_ACCESS_LOCAL_WRITE);
 142                 if (!virtq->umems[i].obj) {
 143                         DRV_LOG(ERR, "Failed to register umem %d for virtq %u.",
 144                                 i, index);
 145                         goto error;
 146                 }
 147                 attr.umems[i].id = virtq->umems[i].obj->umem_id;
 148                 attr.umems[i].offset = 0;
 149                 attr.umems[i].size = virtq->umems[i].size;
 150         }
 151         if (attr.type == MLX5_VIRTQ_TYPE_SPLIT) {
 152                 gpa = mlx5_vdpa_hva_to_gpa(priv->vmem,
 153                                            (uint64_t)(uintptr_t)vq.desc);
 154                 if (!gpa) {
 155                         DRV_LOG(ERR, "Failed to get descriptor ring GPA.");
 156                         goto error;
 157                 }
 158                 attr.desc_addr = gpa;
 159                 gpa = mlx5_vdpa_hva_to_gpa(priv->vmem,
 160                                            (uint64_t)(uintptr_t)vq.used);
 161                 if (!gpa) {
 162                         DRV_LOG(ERR, "Failed to get GPA for used ring.");
 163                         goto error;
 164                 }
 165                 attr.used_addr = gpa;
 166                 gpa = mlx5_vdpa_hva_to_gpa(priv->vmem,
 167                                            (uint64_t)(uintptr_t)vq.avail);
 168                 if (!gpa) {
 169                         DRV_LOG(ERR, "Failed to get GPA for available ring.");
 170                         goto error;
 171                 }
 172                 attr.available_addr = gpa;
 173         }
 174         rte_vhost_get_vring_base(priv->vid, index, &last_avail_idx,
 175                                  &last_used_idx);
 176         DRV_LOG(INFO, "vid %d: Init last_avail_idx=%d, last_used_idx=%d for "
 177                 "virtq %d.", priv->vid, last_avail_idx, last_used_idx, index);
 178         attr.hw_available_index = last_avail_idx;
 179         attr.hw_used_index = last_used_idx;
 180         attr.q_size = vq.size;
 181         attr.mkey = priv->gpa_mkey_index;
 182         attr.tis_id = priv->tis->id;
 183         attr.queue_index = index;
 184         virtq->virtq = mlx5_devx_cmd_create_virtq(priv->ctx, &attr);
 185         if (!virtq->virtq)
 186                 goto error;
 187         return 0;
 188 error:
 189         mlx5_vdpa_virtq_unset(virtq);
 190         return -1;
 191 }
 192
 193 static int
 194 mlx5_vdpa_features_validate(struct mlx5_vdpa_priv *priv)
 195 {
 196         if (priv->features & (1ULL << VIRTIO_F_RING_PACKED)) {
 197                 if (!(priv->caps.virtio_queue_type & (1 <<
 198                                                      MLX5_VIRTQ_TYPE_PACKED))) {
 199                         DRV_LOG(ERR, "Failed to configur PACKED mode for vdev "
 200                                 "%d - it was not reported by HW/driver"
 201                                 " capability.", priv->vid);
 202                         return -ENOTSUP;
 203                 }
 204         }
 205         if (priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO4)) {
 206                 if (!priv->caps.tso_ipv4) {
 207                         DRV_LOG(ERR, "Failed to enable TSO4 for vdev %d - TSO4"
 208                                 " was not reported by HW/driver capability.",
 209                                 priv->vid);
 210                         return -ENOTSUP;
 211                 }
 212         }
 213         if (priv->features & (1ULL << VIRTIO_NET_F_HOST_TSO6)) {
 214                 if (!priv->caps.tso_ipv6) {
 215                         DRV_LOG(ERR, "Failed to enable TSO6 for vdev %d - TSO6"
 216                                 " was not reported by HW/driver capability.",
 217                                 priv->vid);
 218                         return -ENOTSUP;
 219                 }
 220         }
 221         if (priv->features & (1ULL << VIRTIO_NET_F_CSUM)) {
 222                 if (!priv->caps.tx_csum) {
 223                         DRV_LOG(ERR, "Failed to enable CSUM for vdev %d - CSUM"
 224                                 " was not reported by HW/driver capability.",
 225                                 priv->vid);
 226                         return -ENOTSUP;
 227                 }
 228         }
 229         if (priv->features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
 230                 if (!priv->caps.rx_csum) {
 231                         DRV_LOG(ERR, "Failed to enable GUEST CSUM for vdev %d"
 232                                 " GUEST CSUM was not reported by HW/driver "
 233                                 "capability.", priv->vid);
 234                         return -ENOTSUP;
 235                 }
 236         }
 237         if (priv->features & (1ULL << VIRTIO_F_VERSION_1)) {
 238                 if (!priv->caps.virtio_version_1_0) {
 239                         DRV_LOG(ERR, "Failed to enable version 1 for vdev %d "
 240                                 "version 1 was not reported by HW/driver"
 241                                 " capability.", priv->vid);
 242                         return -ENOTSUP;
 243                 }
 244         }
 245         return 0;
 246 }
 247
 248 int
 249 mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
 250 {
 251         struct mlx5_devx_tis_attr tis_attr = {0};
 252         struct mlx5_vdpa_virtq *virtq;
 253         uint32_t i;
 254         uint16_t nr_vring = rte_vhost_get_vring_num(priv->vid);
 255         int ret = rte_vhost_get_negotiated_features(priv->vid, &priv->features);
 256
 257         if (ret || mlx5_vdpa_features_validate(priv)) {
 258                 DRV_LOG(ERR, "Failed to configure negotiated features.");
 259                 return -1;
 260         }
 261         priv->td = mlx5_devx_cmd_create_td(priv->ctx);
 262         if (!priv->td) {
 263                 DRV_LOG(ERR, "Failed to create transport domain.");
 264                 return -rte_errno;
 265         }
 266         tis_attr.transport_domain = priv->td->id;
 267         priv->tis = mlx5_devx_cmd_create_tis(priv->ctx, &tis_attr);
 268         if (!priv->tis) {
 269                 DRV_LOG(ERR, "Failed to create TIS.");
 270                 goto error;
 271         }
 272         for (i = 0; i < nr_vring; i++) {
 273                 virtq = rte_zmalloc(__func__, sizeof(*virtq), 0);
 274                 if (!virtq || mlx5_vdpa_virtq_setup(priv, virtq, i)) {
 275                         if (virtq)
 276                                 rte_free(virtq);
 277                         goto error;
 278                 }
 279                 SLIST_INSERT_HEAD(&priv->virtq_list, virtq, next);
 280         }
 281         priv->nr_virtqs = nr_vring;
 282         return 0;
 283 error:
 284         mlx5_vdpa_virtqs_release(priv);
 285         return -1;
 286 }