drivers/compress/mlx5/mlx5_compress.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2021 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_log.h>
   7 #include <rte_errno.h>
   8 #include <rte_pci.h>
   9 #include <rte_comp.h>
  10 #include <rte_compressdev.h>
  11 #include <rte_compressdev_pmd.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_common_pci.h>
  16 #include <mlx5_devx_cmds.h>
  17 #include <mlx5_common_os.h>
  18 #include <mlx5_prm.h>
  19
  20 #include "mlx5_compress_utils.h"
  21
  22 #define MLX5_COMPRESS_DRIVER_NAME mlx5_compress
  23 #define MLX5_COMPRESS_LOG_NAME    pmd.compress.mlx5
  24
  25 struct mlx5_compress_priv {
  26         TAILQ_ENTRY(mlx5_compress_priv) next;
  27         struct ibv_context *ctx; /* Device context. */
  28         struct rte_pci_device *pci_dev;
  29         struct rte_compressdev *cdev;
  30         void *uar;
  31         uint32_t pdn; /* Protection Domain number. */
  32         uint8_t min_block_size;
  33         /* Minimum huffman block size supported by the device. */
  34         struct ibv_pd *pd;
  35 };
  36
  37 TAILQ_HEAD(mlx5_compress_privs, mlx5_compress_priv) mlx5_compress_priv_list =
  38                                 TAILQ_HEAD_INITIALIZER(mlx5_compress_priv_list);
  39 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  40
  41 int mlx5_compress_logtype;
  42
  43 static struct rte_compressdev_ops mlx5_compress_ops = {
  44         .dev_configure          = NULL,
  45         .dev_start              = NULL,
  46         .dev_stop               = NULL,
  47         .dev_close              = NULL,
  48         .dev_infos_get          = NULL,
  49         .stats_get              = NULL,
  50         .stats_reset            = NULL,
  51         .queue_pair_setup       = NULL,
  52         .queue_pair_release     = NULL,
  53         .private_xform_create   = NULL,
  54         .private_xform_free     = NULL,
  55         .stream_create          = NULL,
  56         .stream_free            = NULL,
  57 };
  58
  59 static struct ibv_device *
  60 mlx5_compress_get_ib_device_match(struct rte_pci_addr *addr)
  61 {
  62         int n;
  63         struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
  64         struct ibv_device *ibv_match = NULL;
  65
  66         if (ibv_list == NULL) {
  67                 rte_errno = ENOSYS;
  68                 return NULL;
  69         }
  70         while (n-- > 0) {
  71                 struct rte_pci_addr paddr;
  72
  73                 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
  74                 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
  75                         continue;
  76                 if (rte_pci_addr_cmp(addr, &paddr) != 0)
  77                         continue;
  78                 ibv_match = ibv_list[n];
  79                 break;
  80         }
  81         if (ibv_match == NULL)
  82                 rte_errno = ENOENT;
  83         mlx5_glue->free_device_list(ibv_list);
  84         return ibv_match;
  85 }
  86
  87 static void
  88 mlx5_compress_hw_global_release(struct mlx5_compress_priv *priv)
  89 {
  90         if (priv->pd != NULL) {
  91                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
  92                 priv->pd = NULL;
  93         }
  94         if (priv->uar != NULL) {
  95                 mlx5_glue->devx_free_uar(priv->uar);
  96                 priv->uar = NULL;
  97         }
  98 }
  99
 100 static int
 101 mlx5_compress_pd_create(struct mlx5_compress_priv *priv)
 102 {
 103 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 104         struct mlx5dv_obj obj;
 105         struct mlx5dv_pd pd_info;
 106         int ret;
 107
 108         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 109         if (priv->pd == NULL) {
 110                 DRV_LOG(ERR, "Failed to allocate PD.");
 111                 return errno ? -errno : -ENOMEM;
 112         }
 113         obj.pd.in = priv->pd;
 114         obj.pd.out = &pd_info;
 115         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 116         if (ret != 0) {
 117                 DRV_LOG(ERR, "Fail to get PD object info.");
 118                 mlx5_glue->dealloc_pd(priv->pd);
 119                 priv->pd = NULL;
 120                 return -errno;
 121         }
 122         priv->pdn = pd_info.pdn;
 123         return 0;
 124 #else
 125         (void)priv;
 126         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 127         return -ENOTSUP;
 128 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 129 }
 130
 131 static int
 132 mlx5_compress_hw_global_prepare(struct mlx5_compress_priv *priv)
 133 {
 134         if (mlx5_compress_pd_create(priv) != 0)
 135                 return -1;
 136         priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
 137         if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
 138             NULL) {
 139                 rte_errno = errno;
 140                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 141                 DRV_LOG(ERR, "Failed to allocate UAR.");
 142                 return -1;
 143         }
 144         return 0;
 145 }
 146
 147 /**
 148  * DPDK callback to register a PCI device.
 149  *
 150  * This function spawns compress device out of a given PCI device.
 151  *
 152  * @param[in] pci_drv
 153  *   PCI driver structure (mlx5_compress_driver).
 154  * @param[in] pci_dev
 155  *   PCI device information.
 156  *
 157  * @return
 158  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 159  *   and rte_errno is set.
 160  */
 161 static int
 162 mlx5_compress_pci_probe(struct rte_pci_driver *pci_drv,
 163                         struct rte_pci_device *pci_dev)
 164 {
 165         struct ibv_device *ibv;
 166         struct rte_compressdev *cdev;
 167         struct ibv_context *ctx;
 168         struct mlx5_compress_priv *priv;
 169         struct mlx5_hca_attr att = { 0 };
 170         struct rte_compressdev_pmd_init_params init_params = {
 171                 .name = "",
 172                 .socket_id = pci_dev->device.numa_node,
 173         };
 174
 175         RTE_SET_USED(pci_drv);
 176         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 177                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 178                 rte_errno = ENOTSUP;
 179                 return -rte_errno;
 180         }
 181         ibv = mlx5_compress_get_ib_device_match(&pci_dev->addr);
 182         if (ibv == NULL) {
 183                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 184                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 185                         pci_dev->addr.bus, pci_dev->addr.devid,
 186                         pci_dev->addr.function);
 187                 return -rte_errno;
 188         }
 189         DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
 190         ctx = mlx5_glue->dv_open_device(ibv);
 191         if (ctx == NULL) {
 192                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 193                 rte_errno = ENODEV;
 194                 return -rte_errno;
 195         }
 196         if (mlx5_devx_cmd_query_hca_attr(ctx, &att) != 0 ||
 197             att.mmo_compress_en == 0 || att.mmo_decompress_en == 0 ||
 198             att.mmo_dma_en == 0) {
 199                 DRV_LOG(ERR, "Not enough capabilities to support compress "
 200                         "operations, maybe old FW/OFED version?");
 201                 claim_zero(mlx5_glue->close_device(ctx));
 202                 rte_errno = ENOTSUP;
 203                 return -ENOTSUP;
 204         }
 205         cdev = rte_compressdev_pmd_create(ibv->name, &pci_dev->device,
 206                                           sizeof(*priv), &init_params);
 207         if (cdev == NULL) {
 208                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
 209                 claim_zero(mlx5_glue->close_device(ctx));
 210                 return -ENODEV;
 211         }
 212         DRV_LOG(INFO,
 213                 "Compress device %s was created successfully.", ibv->name);
 214         cdev->dev_ops = &mlx5_compress_ops;
 215         cdev->dequeue_burst = NULL;
 216         cdev->enqueue_burst = NULL;
 217         cdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED;
 218         priv = cdev->data->dev_private;
 219         priv->ctx = ctx;
 220         priv->pci_dev = pci_dev;
 221         priv->cdev = cdev;
 222         priv->min_block_size = att.compress_min_block_size;
 223         if (mlx5_compress_hw_global_prepare(priv) != 0) {
 224                 rte_compressdev_pmd_destroy(priv->cdev);
 225                 claim_zero(mlx5_glue->close_device(priv->ctx));
 226                 return -1;
 227         }
 228         pthread_mutex_lock(&priv_list_lock);
 229         TAILQ_INSERT_TAIL(&mlx5_compress_priv_list, priv, next);
 230         pthread_mutex_unlock(&priv_list_lock);
 231         return 0;
 232 }
 233
 234 /**
 235  * DPDK callback to remove a PCI device.
 236  *
 237  * This function removes all compress devices belong to a given PCI device.
 238  *
 239  * @param[in] pci_dev
 240  *   Pointer to the PCI device.
 241  *
 242  * @return
 243  *   0 on success, the function cannot fail.
 244  */
 245 static int
 246 mlx5_compress_pci_remove(struct rte_pci_device *pdev)
 247 {
 248         struct mlx5_compress_priv *priv = NULL;
 249
 250         pthread_mutex_lock(&priv_list_lock);
 251         TAILQ_FOREACH(priv, &mlx5_compress_priv_list, next)
 252                 if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
 253                         break;
 254         if (priv)
 255                 TAILQ_REMOVE(&mlx5_compress_priv_list, priv, next);
 256         pthread_mutex_unlock(&priv_list_lock);
 257         if (priv) {
 258                 mlx5_compress_hw_global_release(priv);
 259                 rte_compressdev_pmd_destroy(priv->cdev);
 260                 claim_zero(mlx5_glue->close_device(priv->ctx));
 261         }
 262         return 0;
 263 }
 264
 265 static const struct rte_pci_id mlx5_compress_pci_id_map[] = {
 266         {
 267                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 268                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 269         },
 270         {
 271                 .vendor_id = 0
 272         }
 273 };
 274
 275 static struct mlx5_pci_driver mlx5_compress_driver = {
 276         .driver_class = MLX5_CLASS_COMPRESS,
 277         .pci_driver = {
 278                 .driver = {
 279                         .name = RTE_STR(MLX5_COMPRESS_DRIVER_NAME),
 280                 },
 281                 .id_table = mlx5_compress_pci_id_map,
 282                 .probe = mlx5_compress_pci_probe,
 283                 .remove = mlx5_compress_pci_remove,
 284                 .drv_flags = 0,
 285         },
 286 };
 287
 288 RTE_INIT(rte_mlx5_compress_init)
 289 {
 290         mlx5_common_init();
 291         if (mlx5_glue != NULL)
 292                 mlx5_pci_driver_register(&mlx5_compress_driver);
 293 }
 294
 295 RTE_LOG_REGISTER(mlx5_compress_logtype, MLX5_COMPRESS_LOG_NAME, NOTICE)
 296 RTE_PMD_EXPORT_NAME(MLX5_COMPRESS_DRIVER_NAME, __COUNTER__);
 297 RTE_PMD_REGISTER_PCI_TABLE(MLX5_COMPRESS_DRIVER_NAME, mlx5_compress_pci_id_map);
 298 RTE_PMD_REGISTER_KMOD_DEP(MLX5_COMPRESS_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");