drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_errno.h>
   7 #include <rte_log.h>
   8 #include <rte_pci.h>
   9
  10 #include <mlx5_glue.h>
  11 #include <mlx5_common.h>
  12 #include <mlx5_common_pci.h>
  13 #include <mlx5_devx_cmds.h>
  14 #include <mlx5_common_os.h>
  15
  16 #include "mlx5_crypto_utils.h"
  17 #include "mlx5_crypto.h"
  18
  19 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  20 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  21
  22 #define MLX5_CRYPTO_FEATURE_FLAGS \
  23         RTE_CRYPTODEV_FF_HW_ACCELERATED
  24
  25 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  26                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  27 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  28
  29 int mlx5_crypto_logtype;
  30
  31 uint8_t mlx5_crypto_driver_id;
  32
  33 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  34         {               /* AES XTS */
  35                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  36                 {.sym = {
  37                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  38                         {.cipher = {
  39                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  40                                 .block_size = 16,
  41                                 .key_size = {
  42                                         .min = 32,
  43                                         .max = 64,
  44                                         .increment = 32
  45                                 },
  46                                 .iv_size = {
  47                                         .min = 16,
  48                                         .max = 16,
  49                                         .increment = 0
  50                                 },
  51                                 .dataunit_set =
  52                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  53                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
  54                         }, }
  55                 }, }
  56         },
  57 };
  58
  59 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  60
  61 static const struct rte_driver mlx5_drv = {
  62         .name = mlx5_crypto_drv_name,
  63         .alias = mlx5_crypto_drv_name
  64 };
  65
  66 static struct cryptodev_driver mlx5_cryptodev_driver;
  67
  68 static void
  69 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  70                           struct rte_cryptodev_info *dev_info)
  71 {
  72         RTE_SET_USED(dev);
  73         if (dev_info != NULL) {
  74                 dev_info->driver_id = mlx5_crypto_driver_id;
  75                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
  76                 dev_info->capabilities = mlx5_crypto_caps;
  77                 dev_info->max_nb_queue_pairs = 0;
  78                 dev_info->min_mbuf_headroom_req = 0;
  79                 dev_info->min_mbuf_tailroom_req = 0;
  80                 dev_info->sym.max_nb_sessions = 0;
  81                 /*
  82                  * If 0, the device does not have any limitation in number of
  83                  * sessions that can be used.
  84                  */
  85         }
  86 }
  87
  88 static int
  89 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
  90                           struct rte_cryptodev_config *config)
  91 {
  92         struct mlx5_crypto_priv *priv = dev->data->dev_private;
  93
  94         if (config == NULL) {
  95                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
  96                 return -EINVAL;
  97         }
  98         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
  99                 DRV_LOG(ERR,
 100                         "Disabled symmetric crypto feature is not supported.");
 101                 return -ENOTSUP;
 102         }
 103         if (mlx5_crypto_dek_setup(priv) != 0) {
 104                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 105                 return -ENOMEM;
 106         }
 107         priv->dev_config = *config;
 108         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 109         return 0;
 110 }
 111
 112 static void
 113 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 114 {
 115         RTE_SET_USED(dev);
 116 }
 117
 118 static int
 119 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 120 {
 121         RTE_SET_USED(dev);
 122         return 0;
 123 }
 124
 125 static int
 126 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 127 {
 128         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 129
 130         mlx5_crypto_dek_unset(priv);
 131         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 132         return 0;
 133 }
 134
 135 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 136         .dev_configure                  = mlx5_crypto_dev_configure,
 137         .dev_start                      = mlx5_crypto_dev_start,
 138         .dev_stop                       = mlx5_crypto_dev_stop,
 139         .dev_close                      = mlx5_crypto_dev_close,
 140         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 141         .stats_get                      = NULL,
 142         .stats_reset                    = NULL,
 143         .queue_pair_setup               = NULL,
 144         .queue_pair_release             = NULL,
 145         .sym_session_get_size           = NULL,
 146         .sym_session_configure          = NULL,
 147         .sym_session_clear              = NULL,
 148         .sym_get_raw_dp_ctx_size        = NULL,
 149         .sym_configure_raw_dp_ctx       = NULL,
 150 };
 151
 152 static void
 153 mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
 154 {
 155         if (priv->pd != NULL) {
 156                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 157                 priv->pd = NULL;
 158         }
 159         if (priv->uar != NULL) {
 160                 mlx5_glue->devx_free_uar(priv->uar);
 161                 priv->uar = NULL;
 162         }
 163 }
 164
 165 static int
 166 mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
 167 {
 168 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 169         struct mlx5dv_obj obj;
 170         struct mlx5dv_pd pd_info;
 171         int ret;
 172
 173         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 174         if (priv->pd == NULL) {
 175                 DRV_LOG(ERR, "Failed to allocate PD.");
 176                 return errno ? -errno : -ENOMEM;
 177         }
 178         obj.pd.in = priv->pd;
 179         obj.pd.out = &pd_info;
 180         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 181         if (ret != 0) {
 182                 DRV_LOG(ERR, "Fail to get PD object info.");
 183                 mlx5_glue->dealloc_pd(priv->pd);
 184                 priv->pd = NULL;
 185                 return -errno;
 186         }
 187         priv->pdn = pd_info.pdn;
 188         return 0;
 189 #else
 190         (void)priv;
 191         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 192         return -ENOTSUP;
 193 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 194 }
 195
 196 static int
 197 mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 198 {
 199         if (mlx5_crypto_pd_create(priv) != 0)
 200                 return -1;
 201         priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
 202         if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
 203             NULL) {
 204                 rte_errno = errno;
 205                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 206                 DRV_LOG(ERR, "Failed to allocate UAR.");
 207                 return -1;
 208         }
 209         return 0;
 210 }
 211
 212 /**
 213  * DPDK callback to register a PCI device.
 214  *
 215  * This function spawns crypto device out of a given PCI device.
 216  *
 217  * @param[in] pci_drv
 218  *   PCI driver structure (mlx5_crypto_driver).
 219  * @param[in] pci_dev
 220  *   PCI device information.
 221  *
 222  * @return
 223  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 224  *   and rte_errno is set.
 225  */
 226 static int
 227 mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 228                         struct rte_pci_device *pci_dev)
 229 {
 230         struct ibv_device *ibv;
 231         struct rte_cryptodev *crypto_dev;
 232         struct ibv_context *ctx;
 233         struct mlx5_crypto_priv *priv;
 234         struct mlx5_hca_attr attr = { 0 };
 235         struct rte_cryptodev_pmd_init_params init_params = {
 236                 .name = "",
 237                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 238                 .socket_id = pci_dev->device.numa_node,
 239                 .max_nb_queue_pairs =
 240                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 241         };
 242         RTE_SET_USED(pci_drv);
 243         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 244                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 245                 rte_errno = ENOTSUP;
 246                 return -rte_errno;
 247         }
 248         ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
 249         if (ibv == NULL) {
 250                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 251                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 252                         pci_dev->addr.bus, pci_dev->addr.devid,
 253                         pci_dev->addr.function);
 254                 return -rte_errno;
 255         }
 256         DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
 257         ctx = mlx5_glue->dv_open_device(ibv);
 258         if (ctx == NULL) {
 259                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 260                 rte_errno = ENODEV;
 261                 return -rte_errno;
 262         }
 263         if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
 264             attr.crypto == 0 || attr.aes_xts == 0) {
 265                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 266                         "operations, maybe old FW/OFED version?");
 267                 claim_zero(mlx5_glue->close_device(ctx));
 268                 rte_errno = ENOTSUP;
 269                 return -ENOTSUP;
 270         }
 271         crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 272                                         &init_params);
 273         if (crypto_dev == NULL) {
 274                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
 275                 claim_zero(mlx5_glue->close_device(ctx));
 276                 return -ENODEV;
 277         }
 278         DRV_LOG(INFO,
 279                 "Crypto device %s was created successfully.", ibv->name);
 280         crypto_dev->dev_ops = &mlx5_crypto_ops;
 281         crypto_dev->dequeue_burst = NULL;
 282         crypto_dev->enqueue_burst = NULL;
 283         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 284         crypto_dev->driver_id = mlx5_crypto_driver_id;
 285         priv = crypto_dev->data->dev_private;
 286         priv->ctx = ctx;
 287         priv->pci_dev = pci_dev;
 288         priv->crypto_dev = crypto_dev;
 289         if (mlx5_crypto_hw_global_prepare(priv) != 0) {
 290                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 291                 claim_zero(mlx5_glue->close_device(priv->ctx));
 292                 return -1;
 293         }
 294         pthread_mutex_lock(&priv_list_lock);
 295         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 296         pthread_mutex_unlock(&priv_list_lock);
 297         return 0;
 298 }
 299
 300 static int
 301 mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 302 {
 303         struct mlx5_crypto_priv *priv = NULL;
 304
 305         pthread_mutex_lock(&priv_list_lock);
 306         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 307                 if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
 308                         break;
 309         if (priv)
 310                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 311         pthread_mutex_unlock(&priv_list_lock);
 312         if (priv) {
 313                 mlx5_crypto_hw_global_release(priv);
 314                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 315                 claim_zero(mlx5_glue->close_device(priv->ctx));
 316         }
 317         return 0;
 318 }
 319
 320 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 321                 {
 322                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 323                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 324                 },
 325                 {
 326                         .vendor_id = 0
 327                 }
 328 };
 329
 330 static struct mlx5_pci_driver mlx5_crypto_driver = {
 331         .driver_class = MLX5_CLASS_CRYPTO,
 332         .pci_driver = {
 333                 .driver = {
 334                         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
 335                 },
 336                 .id_table = mlx5_crypto_pci_id_map,
 337                 .probe = mlx5_crypto_pci_probe,
 338                 .remove = mlx5_crypto_pci_remove,
 339                 .drv_flags = 0,
 340         },
 341 };
 342
 343 RTE_INIT(rte_mlx5_crypto_init)
 344 {
 345         mlx5_common_init();
 346         if (mlx5_glue != NULL)
 347                 mlx5_pci_driver_register(&mlx5_crypto_driver);
 348 }
 349
 350 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
 351                                mlx5_crypto_driver_id);
 352
 353 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
 354 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
 355 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
 356 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");