drivers/vdpa/mlx5/mlx5_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <rte_malloc.h>
   5 #include <rte_log.h>
   6 #include <rte_errno.h>
   7 #include <rte_bus_pci.h>
   8
   9 #include <mlx5_glue.h>
  10 #include <mlx5_common.h>
  11 #include <mlx5_devx_cmds.h>
  12 #include <mlx5_prm.h>
  13
  14 #include "mlx5_vdpa_utils.h"
  15 #include "mlx5_vdpa.h"
  16
  17
  18 #define MLX5_VDPA_DEFAULT_FEATURES ((1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
  19                             (1ULL << VIRTIO_F_ANY_LAYOUT) | \
  20                             (1ULL << VIRTIO_NET_F_MQ) | \
  21                             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
  22                             (1ULL << VIRTIO_F_ORDER_PLATFORM) | \
  23                             (1ULL << VHOST_F_LOG_ALL))
  24
  25 #define MLX5_VDPA_PROTOCOL_FEATURES \
  26                             ((1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
  27                              (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
  28                              (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
  29                              (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | \
  30                              (1ULL << VHOST_USER_PROTOCOL_F_MQ))
  31
  32 TAILQ_HEAD(mlx5_vdpa_privs, mlx5_vdpa_priv) priv_list =
  33                                               TAILQ_HEAD_INITIALIZER(priv_list);
  34 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  35 int mlx5_vdpa_logtype;
  36
  37 static struct mlx5_vdpa_priv *
  38 mlx5_vdpa_find_priv_resource_by_did(int did)
  39 {
  40         struct mlx5_vdpa_priv *priv;
  41         int found = 0;
  42
  43         pthread_mutex_lock(&priv_list_lock);
  44         TAILQ_FOREACH(priv, &priv_list, next) {
  45                 if (did == priv->id) {
  46                         found = 1;
  47                         break;
  48                 }
  49         }
  50         pthread_mutex_unlock(&priv_list_lock);
  51         if (!found) {
  52                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  53                 rte_errno = EINVAL;
  54                 return NULL;
  55         }
  56         return priv;
  57 }
  58
  59 static int
  60 mlx5_vdpa_get_queue_num(int did, uint32_t *queue_num)
  61 {
  62         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  63
  64         if (priv == NULL) {
  65                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  66                 return -1;
  67         }
  68         *queue_num = priv->caps.max_num_virtio_queues;
  69         return 0;
  70 }
  71
  72 static int
  73 mlx5_vdpa_get_vdpa_features(int did, uint64_t *features)
  74 {
  75         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
  76
  77         if (priv == NULL) {
  78                 DRV_LOG(ERR, "Invalid device id: %d.", did);
  79                 return -1;
  80         }
  81         *features = MLX5_VDPA_DEFAULT_FEATURES;
  82         if (priv->caps.virtio_queue_type & (1 << MLX5_VIRTQ_TYPE_PACKED))
  83                 *features |= (1ULL << VIRTIO_F_RING_PACKED);
  84         if (priv->caps.tso_ipv4)
  85                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
  86         if (priv->caps.tso_ipv6)
  87                 *features |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
  88         if (priv->caps.tx_csum)
  89                 *features |= (1ULL << VIRTIO_NET_F_CSUM);
  90         if (priv->caps.rx_csum)
  91                 *features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
  92         if (priv->caps.virtio_version_1_0)
  93                 *features |= (1ULL << VIRTIO_F_VERSION_1);
  94         return 0;
  95 }
  96
  97 static int
  98 mlx5_vdpa_get_protocol_features(int did, uint64_t *features)
  99 {
 100         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 101
 102         if (priv == NULL) {
 103                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 104                 return -1;
 105         }
 106         *features = MLX5_VDPA_PROTOCOL_FEATURES;
 107         return 0;
 108 }
 109
 110 static int
 111 mlx5_vdpa_set_vring_state(int vid, int vring, int state)
 112 {
 113         int did = rte_vhost_get_vdpa_device_id(vid);
 114         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 115         struct mlx5_vdpa_virtq *virtq = NULL;
 116
 117         if (priv == NULL) {
 118                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 119                 return -EINVAL;
 120         }
 121         SLIST_FOREACH(virtq, &priv->virtq_list, next)
 122                 if (virtq->index == vring)
 123                         break;
 124         if (!virtq) {
 125                 DRV_LOG(ERR, "Invalid or unconfigured vring id: %d.", vring);
 126                 return -EINVAL;
 127         }
 128         return mlx5_vdpa_virtq_enable(virtq, state);
 129 }
 130
 131 static int
 132 mlx5_vdpa_features_set(int vid)
 133 {
 134         int did = rte_vhost_get_vdpa_device_id(vid);
 135         struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
 136         uint64_t log_base, log_size;
 137         uint64_t features;
 138         int ret;
 139
 140         if (priv == NULL) {
 141                 DRV_LOG(ERR, "Invalid device id: %d.", did);
 142                 return -EINVAL;
 143         }
 144         ret = rte_vhost_get_negotiated_features(vid, &features);
 145         if (ret) {
 146                 DRV_LOG(ERR, "Failed to get negotiated features.");
 147                 return ret;
 148         }
 149         if (RTE_VHOST_NEED_LOG(features)) {
 150                 ret = rte_vhost_get_log_base(vid, &log_base, &log_size);
 151                 if (ret) {
 152                         DRV_LOG(ERR, "Failed to get log base.");
 153                         return ret;
 154                 }
 155                 ret = mlx5_vdpa_dirty_bitmap_set(priv, log_base, log_size);
 156                 if (ret) {
 157                         DRV_LOG(ERR, "Failed to set dirty bitmap.");
 158                         return ret;
 159                 }
 160                 DRV_LOG(INFO, "mlx5 vdpa: enabling dirty logging...");
 161                 ret = mlx5_vdpa_logging_enable(priv, 1);
 162                 if (ret) {
 163                         DRV_LOG(ERR, "Failed t enable dirty logging.");
 164                         return ret;
 165                 }
 166         }
 167         return 0;
 168 }
 169
 170 static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 171         .get_queue_num = mlx5_vdpa_get_queue_num,
 172         .get_features = mlx5_vdpa_get_vdpa_features,
 173         .get_protocol_features = mlx5_vdpa_get_protocol_features,
 174         .dev_conf = NULL,
 175         .dev_close = NULL,
 176         .set_vring_state = mlx5_vdpa_set_vring_state,
 177         .set_features = mlx5_vdpa_features_set,
 178         .migration_done = NULL,
 179         .get_vfio_group_fd = NULL,
 180         .get_vfio_device_fd = NULL,
 181         .get_notify_area = NULL,
 182 };
 183
 184 /**
 185  * DPDK callback to register a PCI device.
 186  *
 187  * This function spawns vdpa device out of a given PCI device.
 188  *
 189  * @param[in] pci_drv
 190  *   PCI driver structure (mlx5_vpda_driver).
 191  * @param[in] pci_dev
 192  *   PCI device information.
 193  *
 194  * @return
 195  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 196  *   and rte_errno is set.
 197  */
 198 static int
 199 mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 200                     struct rte_pci_device *pci_dev __rte_unused)
 201 {
 202         struct ibv_device **ibv_list;
 203         struct ibv_device *ibv_match = NULL;
 204         struct mlx5_vdpa_priv *priv = NULL;
 205         struct ibv_context *ctx = NULL;
 206         struct mlx5_hca_attr attr;
 207         int ret;
 208
 209         if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_VDPA) {
 210                 DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
 211                         " driver.");
 212                 return 1;
 213         }
 214         errno = 0;
 215         ibv_list = mlx5_glue->get_device_list(&ret);
 216         if (!ibv_list) {
 217                 rte_errno = ENOSYS;
 218                 DRV_LOG(ERR, "Failed to get device list, is ib_uverbs loaded?");
 219                 return -rte_errno;
 220         }
 221         while (ret-- > 0) {
 222                 struct rte_pci_addr pci_addr;
 223
 224                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[ret]->name);
 225                 if (mlx5_dev_to_pci_addr(ibv_list[ret]->ibdev_path, &pci_addr))
 226                         continue;
 227                 if (pci_dev->addr.domain != pci_addr.domain ||
 228                     pci_dev->addr.bus != pci_addr.bus ||
 229                     pci_dev->addr.devid != pci_addr.devid ||
 230                     pci_dev->addr.function != pci_addr.function)
 231                         continue;
 232                 DRV_LOG(INFO, "PCI information matches for device \"%s\".",
 233                         ibv_list[ret]->name);
 234                 ibv_match = ibv_list[ret];
 235                 break;
 236         }
 237         mlx5_glue->free_device_list(ibv_list);
 238         if (!ibv_match) {
 239                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 240                         "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 ".",
 241                         pci_dev->addr.domain, pci_dev->addr.bus,
 242                         pci_dev->addr.devid, pci_dev->addr.function);
 243                 rte_errno = ENOENT;
 244                 return -rte_errno;
 245         }
 246         ctx = mlx5_glue->dv_open_device(ibv_match);
 247         if (!ctx) {
 248                 DRV_LOG(ERR, "Failed to open IB device \"%s\".",
 249                         ibv_match->name);
 250                 rte_errno = ENODEV;
 251                 return -rte_errno;
 252         }
 253         priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv),
 254                            RTE_CACHE_LINE_SIZE);
 255         if (!priv) {
 256                 DRV_LOG(ERR, "Failed to allocate private memory.");
 257                 rte_errno = ENOMEM;
 258                 goto error;
 259         }
 260         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 261         if (ret) {
 262                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 263                 rte_errno = ENOTSUP;
 264                 goto error;
 265         } else {
 266                 if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
 267                         DRV_LOG(ERR, "Not enough capabilities to support vdpa,"
 268                                 " maybe old FW/OFED version?");
 269                         rte_errno = ENOTSUP;
 270                         goto error;
 271                 }
 272                 priv->caps = attr.vdpa;
 273                 priv->log_max_rqt_size = attr.log_max_rqt_size;
 274         }
 275         priv->ctx = ctx;
 276         priv->dev_addr.pci_addr = pci_dev->addr;
 277         priv->dev_addr.type = PCI_ADDR;
 278         priv->id = rte_vdpa_register_device(&priv->dev_addr, &mlx5_vdpa_ops);
 279         if (priv->id < 0) {
 280                 DRV_LOG(ERR, "Failed to register vDPA device.");
 281                 rte_errno = rte_errno ? rte_errno : EINVAL;
 282                 goto error;
 283         }
 284         SLIST_INIT(&priv->mr_list);
 285         SLIST_INIT(&priv->virtq_list);
 286         pthread_mutex_lock(&priv_list_lock);
 287         TAILQ_INSERT_TAIL(&priv_list, priv, next);
 288         pthread_mutex_unlock(&priv_list_lock);
 289         return 0;
 290
 291 error:
 292         if (priv)
 293                 rte_free(priv);
 294         if (ctx)
 295                 mlx5_glue->close_device(ctx);
 296         return -rte_errno;
 297 }
 298
 299 /**
 300  * DPDK callback to remove a PCI device.
 301  *
 302  * This function removes all vDPA devices belong to a given PCI device.
 303  *
 304  * @param[in] pci_dev
 305  *   Pointer to the PCI device.
 306  *
 307  * @return
 308  *   0 on success, the function cannot fail.
 309  */
 310 static int
 311 mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
 312 {
 313         struct mlx5_vdpa_priv *priv = NULL;
 314         int found = 0;
 315
 316         pthread_mutex_lock(&priv_list_lock);
 317         TAILQ_FOREACH(priv, &priv_list, next) {
 318                 if (memcmp(&priv->dev_addr.pci_addr, &pci_dev->addr,
 319                            sizeof(pci_dev->addr)) == 0) {
 320                         found = 1;
 321                         break;
 322                 }
 323         }
 324         if (found) {
 325                 TAILQ_REMOVE(&priv_list, priv, next);
 326                 mlx5_glue->close_device(priv->ctx);
 327                 rte_free(priv);
 328         }
 329         pthread_mutex_unlock(&priv_list_lock);
 330         return 0;
 331 }
 332
 333 static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
 334         {
 335                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 336                                PCI_DEVICE_ID_MELLANOX_CONNECTX5BF)
 337         },
 338         {
 339                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 340                                PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF)
 341         },
 342         {
 343                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 344                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 345         },
 346         {
 347                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 348                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF)
 349         },
 350         {
 351                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 352                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
 353         },
 354         {
 355                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 356                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF)
 357         },
 358         {
 359                 .vendor_id = 0
 360         }
 361 };
 362
 363 static struct rte_pci_driver mlx5_vdpa_driver = {
 364         .driver = {
 365                 .name = "mlx5_vdpa",
 366         },
 367         .id_table = mlx5_vdpa_pci_id_map,
 368         .probe = mlx5_vdpa_pci_probe,
 369         .remove = mlx5_vdpa_pci_remove,
 370         .drv_flags = 0,
 371 };
 372
 373 /**
 374  * Driver initialization routine.
 375  */
 376 RTE_INIT(rte_mlx5_vdpa_init)
 377 {
 378         /* Initialize common log type. */
 379         mlx5_vdpa_logtype = rte_log_register("pmd.vdpa.mlx5");
 380         if (mlx5_vdpa_logtype >= 0)
 381                 rte_log_set_level(mlx5_vdpa_logtype, RTE_LOG_NOTICE);
 382         if (mlx5_glue)
 383                 rte_pci_register(&mlx5_vdpa_driver);
 384 }
 385
 386 RTE_PMD_EXPORT_NAME(net_mlx5_vdpa, __COUNTER__);
 387 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5_vdpa, mlx5_vdpa_pci_id_map);
 388 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5_vdpa, "* ib_uverbs & mlx5_core & mlx5_ib");