drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_errno.h>
   8 #include <rte_log.h>
   9 #include <rte_pci.h>
  10 #include <rte_memory.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_common.h>
  14 #include <mlx5_common_pci.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 1024
  24
  25 #define MLX5_CRYPTO_FEATURE_FLAGS \
  26         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  27          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  28          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  29
  30 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  31                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  32 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  33
  34 int mlx5_crypto_logtype;
  35
  36 uint8_t mlx5_crypto_driver_id;
  37
  38 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  39         {               /* AES XTS */
  40                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  41                 {.sym = {
  42                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  43                         {.cipher = {
  44                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  45                                 .block_size = 16,
  46                                 .key_size = {
  47                                         .min = 32,
  48                                         .max = 64,
  49                                         .increment = 32
  50                                 },
  51                                 .iv_size = {
  52                                         .min = 16,
  53                                         .max = 16,
  54                                         .increment = 0
  55                                 },
  56                                 .dataunit_set =
  57                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  58                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
  59                         }, }
  60                 }, }
  61         },
  62 };
  63
  64 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  65
  66 static const struct rte_driver mlx5_drv = {
  67         .name = mlx5_crypto_drv_name,
  68         .alias = mlx5_crypto_drv_name
  69 };
  70
  71 static struct cryptodev_driver mlx5_cryptodev_driver;
  72
  73 struct mlx5_crypto_session {
  74         uint32_t bs_bpt_eo_es;
  75         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  76          * saved in big endian format.
  77          */
  78         uint32_t bsp_res;
  79         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  80          * endian format.
  81          */
  82         uint32_t iv_offset:16;
  83         /**< Starting point for Initialisation Vector. */
  84         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  85         uint32_t dek_id; /**< DEK ID */
  86 } __rte_packed;
  87
  88 static void
  89 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  90                           struct rte_cryptodev_info *dev_info)
  91 {
  92         RTE_SET_USED(dev);
  93         if (dev_info != NULL) {
  94                 dev_info->driver_id = mlx5_crypto_driver_id;
  95                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
  96                 dev_info->capabilities = mlx5_crypto_caps;
  97                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
  98                 dev_info->min_mbuf_headroom_req = 0;
  99                 dev_info->min_mbuf_tailroom_req = 0;
 100                 dev_info->sym.max_nb_sessions = 0;
 101                 /*
 102                  * If 0, the device does not have any limitation in number of
 103                  * sessions that can be used.
 104                  */
 105         }
 106 }
 107
 108 static int
 109 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 110                           struct rte_cryptodev_config *config)
 111 {
 112         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 113
 114         if (config == NULL) {
 115                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 116                 return -EINVAL;
 117         }
 118         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 119                 DRV_LOG(ERR,
 120                         "Disabled symmetric crypto feature is not supported.");
 121                 return -ENOTSUP;
 122         }
 123         if (mlx5_crypto_dek_setup(priv) != 0) {
 124                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 125                 return -ENOMEM;
 126         }
 127         priv->dev_config = *config;
 128         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 129         return 0;
 130 }
 131
 132 static void
 133 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 134 {
 135         RTE_SET_USED(dev);
 136 }
 137
 138 static int
 139 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 140 {
 141         RTE_SET_USED(dev);
 142         return 0;
 143 }
 144
 145 static int
 146 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 147 {
 148         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 149
 150         mlx5_crypto_dek_unset(priv);
 151         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 152         return 0;
 153 }
 154
 155 static unsigned int
 156 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 157 {
 158         return sizeof(struct mlx5_crypto_session);
 159 }
 160
 161 static int
 162 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 163                                   struct rte_crypto_sym_xform *xform,
 164                                   struct rte_cryptodev_sym_session *session,
 165                                   struct rte_mempool *mp)
 166 {
 167         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 168         struct mlx5_crypto_session *sess_private_data;
 169         struct rte_crypto_cipher_xform *cipher;
 170         uint8_t encryption_order;
 171         int ret;
 172
 173         if (unlikely(xform->next != NULL)) {
 174                 DRV_LOG(ERR, "Xform next is not supported.");
 175                 return -ENOTSUP;
 176         }
 177         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 178                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 179                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 180                 return -ENOTSUP;
 181         }
 182         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 183         if (ret != 0) {
 184                 DRV_LOG(ERR,
 185                         "Failed to get session %p private data from mempool.",
 186                         sess_private_data);
 187                 return -ENOMEM;
 188         }
 189         cipher = &xform->cipher;
 190         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 191         if (sess_private_data->dek == NULL) {
 192                 rte_mempool_put(mp, sess_private_data);
 193                 DRV_LOG(ERR, "Failed to prepare dek.");
 194                 return -ENOMEM;
 195         }
 196         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 197                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 198         else
 199                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 200         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 201                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 202                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 203                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 204                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 205         switch (xform->cipher.dataunit_len) {
 206         case 0:
 207                 sess_private_data->bsp_res = 0;
 208                 break;
 209         case 512:
 210                 sess_private_data->bsp_res = rte_cpu_to_be_32
 211                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 212                                              MLX5_BLOCK_SIZE_OFFSET);
 213                 break;
 214         case 4096:
 215                 sess_private_data->bsp_res = rte_cpu_to_be_32
 216                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 217                                              MLX5_BLOCK_SIZE_OFFSET);
 218                 break;
 219         default:
 220                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 221                 return -ENOTSUP;
 222         }
 223         sess_private_data->iv_offset = cipher->iv.offset;
 224         sess_private_data->dek_id =
 225                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 226                                          0xffffff);
 227         set_sym_session_private_data(session, dev->driver_id,
 228                                      sess_private_data);
 229         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 230         return 0;
 231 }
 232
 233 static void
 234 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 235                               struct rte_cryptodev_sym_session *sess)
 236 {
 237         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 238         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 239                                                                 dev->driver_id);
 240
 241         if (unlikely(spriv == NULL)) {
 242                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 243                 return;
 244         }
 245         mlx5_crypto_dek_destroy(priv, spriv->dek);
 246         set_sym_session_private_data(sess, dev->driver_id, NULL);
 247         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 248         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 249 }
 250
 251 static int
 252 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 253 {
 254         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 255
 256         if (qp->qp_obj != NULL)
 257                 claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
 258         if (qp->umem_obj != NULL)
 259                 claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 260         if (qp->umem_buf != NULL)
 261                 rte_free(qp->umem_buf);
 262         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 263         mlx5_devx_cq_destroy(&qp->cq_obj);
 264         rte_free(qp);
 265         dev->data->queue_pairs[qp_id] = NULL;
 266         return 0;
 267 }
 268
 269 static int
 270 mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 271 {
 272         /*
 273          * In Order to configure self loopback, when calling these functions the
 274          * remote QP id that is used is the id of the same QP.
 275          */
 276         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
 277                                           qp->qp_obj->id)) {
 278                 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
 279                         rte_errno);
 280                 return -1;
 281         }
 282         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
 283                                           qp->qp_obj->id)) {
 284                 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
 285                         rte_errno);
 286                 return -1;
 287         }
 288         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
 289                                           qp->qp_obj->id)) {
 290                 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
 291                         rte_errno);
 292                 return -1;
 293         }
 294         return 0;
 295 }
 296
 297 static int
 298 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 299                              const struct rte_cryptodev_qp_conf *qp_conf,
 300                              int socket_id)
 301 {
 302         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 303         struct mlx5_devx_qp_attr attr = {0};
 304         struct mlx5_crypto_qp *qp;
 305         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 306         uint32_t umem_size = RTE_BIT32(log_nb_desc) *
 307                               MLX5_CRYPTO_WQE_SET_SIZE +
 308                               sizeof(*qp->db_rec) * 2;
 309         uint32_t alloc_size = sizeof(*qp);
 310         struct mlx5_devx_cq_attr cq_attr = {
 311                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
 312         };
 313
 314         if (dev->data->queue_pairs[qp_id] != NULL)
 315                 mlx5_crypto_queue_pair_release(dev, qp_id);
 316         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 317         alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
 318         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 319                                 socket_id);
 320         if (qp == NULL) {
 321                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 322                 rte_errno = ENOMEM;
 323                 return -rte_errno;
 324         }
 325         if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
 326                                 &cq_attr, socket_id) != 0) {
 327                 DRV_LOG(ERR, "Failed to create CQ.");
 328                 goto error;
 329         }
 330         qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
 331         if (qp->umem_buf == NULL) {
 332                 DRV_LOG(ERR, "Failed to allocate QP umem.");
 333                 rte_errno = ENOMEM;
 334                 goto error;
 335         }
 336         qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 337                                                (void *)(uintptr_t)qp->umem_buf,
 338                                                umem_size,
 339                                                IBV_ACCESS_LOCAL_WRITE);
 340         if (qp->umem_obj == NULL) {
 341                 DRV_LOG(ERR, "Failed to register QP umem.");
 342                 goto error;
 343         }
 344         if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
 345                                priv->dev_config.socket_id) != 0) {
 346                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 347                         (uint32_t)qp_id);
 348                 rte_errno = ENOMEM;
 349                 goto error;
 350         }
 351         qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 352         attr.pd = priv->pdn;
 353         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 354         attr.cqn = qp->cq_obj.cq->id;
 355         attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
 356         attr.rq_size =  0;
 357         attr.sq_size = RTE_BIT32(log_nb_desc);
 358         attr.dbr_umem_valid = 1;
 359         attr.wq_umem_id = qp->umem_obj->umem_id;
 360         attr.wq_umem_offset = 0;
 361         attr.dbr_umem_id = qp->umem_obj->umem_id;
 362         attr.dbr_address = RTE_BIT64(log_nb_desc) *
 363                            MLX5_CRYPTO_WQE_SET_SIZE;
 364         qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 365         if (qp->qp_obj == NULL) {
 366                 DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
 367                 goto error;
 368         }
 369         qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 370         if (mlx5_crypto_qp2rts(qp))
 371                 goto error;
 372         qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
 373                                                            RTE_CACHE_LINE_SIZE);
 374         dev->data->queue_pairs[qp_id] = qp;
 375         return 0;
 376 error:
 377         mlx5_crypto_queue_pair_release(dev, qp_id);
 378         return -1;
 379 }
 380
 381 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 382         .dev_configure                  = mlx5_crypto_dev_configure,
 383         .dev_start                      = mlx5_crypto_dev_start,
 384         .dev_stop                       = mlx5_crypto_dev_stop,
 385         .dev_close                      = mlx5_crypto_dev_close,
 386         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 387         .stats_get                      = NULL,
 388         .stats_reset                    = NULL,
 389         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 390         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 391         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 392         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 393         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 394         .sym_get_raw_dp_ctx_size        = NULL,
 395         .sym_configure_raw_dp_ctx       = NULL,
 396 };
 397
 398 static void
 399 mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
 400 {
 401         if (priv->pd != NULL) {
 402                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 403                 priv->pd = NULL;
 404         }
 405         if (priv->uar != NULL) {
 406                 mlx5_glue->devx_free_uar(priv->uar);
 407                 priv->uar = NULL;
 408         }
 409 }
 410
 411 static int
 412 mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
 413 {
 414 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 415         struct mlx5dv_obj obj;
 416         struct mlx5dv_pd pd_info;
 417         int ret;
 418
 419         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 420         if (priv->pd == NULL) {
 421                 DRV_LOG(ERR, "Failed to allocate PD.");
 422                 return errno ? -errno : -ENOMEM;
 423         }
 424         obj.pd.in = priv->pd;
 425         obj.pd.out = &pd_info;
 426         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 427         if (ret != 0) {
 428                 DRV_LOG(ERR, "Fail to get PD object info.");
 429                 mlx5_glue->dealloc_pd(priv->pd);
 430                 priv->pd = NULL;
 431                 return -errno;
 432         }
 433         priv->pdn = pd_info.pdn;
 434         return 0;
 435 #else
 436         (void)priv;
 437         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 438         return -ENOTSUP;
 439 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 440 }
 441
 442 static int
 443 mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 444 {
 445         if (mlx5_crypto_pd_create(priv) != 0)
 446                 return -1;
 447         priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
 448         if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
 449             NULL) {
 450                 rte_errno = errno;
 451                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 452                 DRV_LOG(ERR, "Failed to allocate UAR.");
 453                 return -1;
 454         }
 455         return 0;
 456 }
 457
 458 /**
 459  * Callback for memory event.
 460  *
 461  * @param event_type
 462  *   Memory event type.
 463  * @param addr
 464  *   Address of memory.
 465  * @param len
 466  *   Size of memory.
 467  */
 468 static void
 469 mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 470                             size_t len, void *arg __rte_unused)
 471 {
 472         struct mlx5_crypto_priv *priv;
 473
 474         /* Must be called from the primary process. */
 475         MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
 476         switch (event_type) {
 477         case RTE_MEM_EVENT_FREE:
 478                 pthread_mutex_lock(&priv_list_lock);
 479                 /* Iterate all the existing mlx5 devices. */
 480                 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 481                         mlx5_free_mr_by_addr(&priv->mr_scache,
 482                                              priv->ctx->device->name,
 483                                              addr, len);
 484                 pthread_mutex_unlock(&priv_list_lock);
 485                 break;
 486         case RTE_MEM_EVENT_ALLOC:
 487         default:
 488                 break;
 489         }
 490 }
 491
 492 /**
 493  * DPDK callback to register a PCI device.
 494  *
 495  * This function spawns crypto device out of a given PCI device.
 496  *
 497  * @param[in] pci_drv
 498  *   PCI driver structure (mlx5_crypto_driver).
 499  * @param[in] pci_dev
 500  *   PCI device information.
 501  *
 502  * @return
 503  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 504  *   and rte_errno is set.
 505  */
 506 static int
 507 mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 508                         struct rte_pci_device *pci_dev)
 509 {
 510         struct ibv_device *ibv;
 511         struct rte_cryptodev *crypto_dev;
 512         struct ibv_context *ctx;
 513         struct mlx5_crypto_priv *priv;
 514         struct mlx5_hca_attr attr = { 0 };
 515         struct rte_cryptodev_pmd_init_params init_params = {
 516                 .name = "",
 517                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 518                 .socket_id = pci_dev->device.numa_node,
 519                 .max_nb_queue_pairs =
 520                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 521         };
 522         RTE_SET_USED(pci_drv);
 523         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 524                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 525                 rte_errno = ENOTSUP;
 526                 return -rte_errno;
 527         }
 528         ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
 529         if (ibv == NULL) {
 530                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 531                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 532                         pci_dev->addr.bus, pci_dev->addr.devid,
 533                         pci_dev->addr.function);
 534                 return -rte_errno;
 535         }
 536         DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
 537         ctx = mlx5_glue->dv_open_device(ibv);
 538         if (ctx == NULL) {
 539                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 540                 rte_errno = ENODEV;
 541                 return -rte_errno;
 542         }
 543         if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
 544             attr.crypto == 0 || attr.aes_xts == 0) {
 545                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 546                         "operations, maybe old FW/OFED version?");
 547                 claim_zero(mlx5_glue->close_device(ctx));
 548                 rte_errno = ENOTSUP;
 549                 return -ENOTSUP;
 550         }
 551         crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 552                                         &init_params);
 553         if (crypto_dev == NULL) {
 554                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
 555                 claim_zero(mlx5_glue->close_device(ctx));
 556                 return -ENODEV;
 557         }
 558         DRV_LOG(INFO,
 559                 "Crypto device %s was created successfully.", ibv->name);
 560         crypto_dev->dev_ops = &mlx5_crypto_ops;
 561         crypto_dev->dequeue_burst = NULL;
 562         crypto_dev->enqueue_burst = NULL;
 563         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 564         crypto_dev->driver_id = mlx5_crypto_driver_id;
 565         priv = crypto_dev->data->dev_private;
 566         priv->ctx = ctx;
 567         priv->pci_dev = pci_dev;
 568         priv->crypto_dev = crypto_dev;
 569         if (mlx5_crypto_hw_global_prepare(priv) != 0) {
 570                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 571                 claim_zero(mlx5_glue->close_device(priv->ctx));
 572                 return -1;
 573         }
 574         if (mlx5_mr_btree_init(&priv->mr_scache.cache,
 575                              MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
 576                 DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
 577                 mlx5_crypto_hw_global_release(priv);
 578                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 579                 claim_zero(mlx5_glue->close_device(priv->ctx));
 580                 rte_errno = ENOMEM;
 581                 return -rte_errno;
 582         }
 583         priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 584         priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 585         /* Register callback function for global shared MR cache management. */
 586         if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 587                 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
 588                                                 mlx5_crypto_mr_mem_event_cb,
 589                                                 NULL);
 590         pthread_mutex_lock(&priv_list_lock);
 591         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 592         pthread_mutex_unlock(&priv_list_lock);
 593         return 0;
 594 }
 595
 596 static int
 597 mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 598 {
 599         struct mlx5_crypto_priv *priv = NULL;
 600
 601         pthread_mutex_lock(&priv_list_lock);
 602         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 603                 if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
 604                         break;
 605         if (priv)
 606                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 607         pthread_mutex_unlock(&priv_list_lock);
 608         if (priv) {
 609                 if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 610                         rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
 611                                                           NULL);
 612                 mlx5_mr_release_cache(&priv->mr_scache);
 613                 mlx5_crypto_hw_global_release(priv);
 614                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 615                 claim_zero(mlx5_glue->close_device(priv->ctx));
 616         }
 617         return 0;
 618 }
 619
 620 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 621                 {
 622                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 623                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 624                 },
 625                 {
 626                         .vendor_id = 0
 627                 }
 628 };
 629
 630 static struct mlx5_pci_driver mlx5_crypto_driver = {
 631         .driver_class = MLX5_CLASS_CRYPTO,
 632         .pci_driver = {
 633                 .driver = {
 634                         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
 635                 },
 636                 .id_table = mlx5_crypto_pci_id_map,
 637                 .probe = mlx5_crypto_pci_probe,
 638                 .remove = mlx5_crypto_pci_remove,
 639                 .drv_flags = 0,
 640         },
 641 };
 642
 643 RTE_INIT(rte_mlx5_crypto_init)
 644 {
 645         mlx5_common_init();
 646         if (mlx5_glue != NULL)
 647                 mlx5_pci_driver_register(&mlx5_crypto_driver);
 648 }
 649
 650 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
 651                                mlx5_crypto_driver_id);
 652
 653 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
 654 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
 655 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
 656 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");