drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_errno.h>
   8 #include <rte_log.h>
   9 #include <rte_pci.h>
  10 #include <rte_memory.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_common.h>
  14 #include <mlx5_common_pci.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 1024
  24 #define MLX5_CRYPTO_MAX_SEGS 56
  25
  26 #define MLX5_CRYPTO_FEATURE_FLAGS \
  27         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  28          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  29          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  30
  31 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  32                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  33 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  34
  35 int mlx5_crypto_logtype;
  36
  37 uint8_t mlx5_crypto_driver_id;
  38
  39 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  40         {               /* AES XTS */
  41                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  42                 {.sym = {
  43                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  44                         {.cipher = {
  45                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  46                                 .block_size = 16,
  47                                 .key_size = {
  48                                         .min = 32,
  49                                         .max = 64,
  50                                         .increment = 32
  51                                 },
  52                                 .iv_size = {
  53                                         .min = 16,
  54                                         .max = 16,
  55                                         .increment = 0
  56                                 },
  57                                 .dataunit_set =
  58                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  59                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
  60                         }, }
  61                 }, }
  62         },
  63 };
  64
  65 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  66
  67 static const struct rte_driver mlx5_drv = {
  68         .name = mlx5_crypto_drv_name,
  69         .alias = mlx5_crypto_drv_name
  70 };
  71
  72 static struct cryptodev_driver mlx5_cryptodev_driver;
  73
  74 struct mlx5_crypto_session {
  75         uint32_t bs_bpt_eo_es;
  76         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  77          * saved in big endian format.
  78          */
  79         uint32_t bsp_res;
  80         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  81          * endian format.
  82          */
  83         uint32_t iv_offset:16;
  84         /**< Starting point for Initialisation Vector. */
  85         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  86         uint32_t dek_id; /**< DEK ID */
  87 } __rte_packed;
  88
  89 static void
  90 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  91                           struct rte_cryptodev_info *dev_info)
  92 {
  93         RTE_SET_USED(dev);
  94         if (dev_info != NULL) {
  95                 dev_info->driver_id = mlx5_crypto_driver_id;
  96                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
  97                 dev_info->capabilities = mlx5_crypto_caps;
  98                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
  99                 dev_info->min_mbuf_headroom_req = 0;
 100                 dev_info->min_mbuf_tailroom_req = 0;
 101                 dev_info->sym.max_nb_sessions = 0;
 102                 /*
 103                  * If 0, the device does not have any limitation in number of
 104                  * sessions that can be used.
 105                  */
 106         }
 107 }
 108
 109 static int
 110 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 111                           struct rte_cryptodev_config *config)
 112 {
 113         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 114
 115         if (config == NULL) {
 116                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 117                 return -EINVAL;
 118         }
 119         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 120                 DRV_LOG(ERR,
 121                         "Disabled symmetric crypto feature is not supported.");
 122                 return -ENOTSUP;
 123         }
 124         if (mlx5_crypto_dek_setup(priv) != 0) {
 125                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 126                 return -ENOMEM;
 127         }
 128         priv->dev_config = *config;
 129         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 130         return 0;
 131 }
 132
 133 static void
 134 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 135 {
 136         RTE_SET_USED(dev);
 137 }
 138
 139 static int
 140 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 141 {
 142         RTE_SET_USED(dev);
 143         return 0;
 144 }
 145
 146 static int
 147 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 148 {
 149         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 150
 151         mlx5_crypto_dek_unset(priv);
 152         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 153         return 0;
 154 }
 155
 156 static unsigned int
 157 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 158 {
 159         return sizeof(struct mlx5_crypto_session);
 160 }
 161
 162 static int
 163 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 164                                   struct rte_crypto_sym_xform *xform,
 165                                   struct rte_cryptodev_sym_session *session,
 166                                   struct rte_mempool *mp)
 167 {
 168         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 169         struct mlx5_crypto_session *sess_private_data;
 170         struct rte_crypto_cipher_xform *cipher;
 171         uint8_t encryption_order;
 172         int ret;
 173
 174         if (unlikely(xform->next != NULL)) {
 175                 DRV_LOG(ERR, "Xform next is not supported.");
 176                 return -ENOTSUP;
 177         }
 178         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 179                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 180                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 181                 return -ENOTSUP;
 182         }
 183         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 184         if (ret != 0) {
 185                 DRV_LOG(ERR,
 186                         "Failed to get session %p private data from mempool.",
 187                         sess_private_data);
 188                 return -ENOMEM;
 189         }
 190         cipher = &xform->cipher;
 191         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 192         if (sess_private_data->dek == NULL) {
 193                 rte_mempool_put(mp, sess_private_data);
 194                 DRV_LOG(ERR, "Failed to prepare dek.");
 195                 return -ENOMEM;
 196         }
 197         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 198                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 199         else
 200                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 201         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 202                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 203                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 204                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 205                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 206         switch (xform->cipher.dataunit_len) {
 207         case 0:
 208                 sess_private_data->bsp_res = 0;
 209                 break;
 210         case 512:
 211                 sess_private_data->bsp_res = rte_cpu_to_be_32
 212                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 213                                              MLX5_BLOCK_SIZE_OFFSET);
 214                 break;
 215         case 4096:
 216                 sess_private_data->bsp_res = rte_cpu_to_be_32
 217                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 218                                              MLX5_BLOCK_SIZE_OFFSET);
 219                 break;
 220         default:
 221                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 222                 return -ENOTSUP;
 223         }
 224         sess_private_data->iv_offset = cipher->iv.offset;
 225         sess_private_data->dek_id =
 226                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 227                                          0xffffff);
 228         set_sym_session_private_data(session, dev->driver_id,
 229                                      sess_private_data);
 230         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 231         return 0;
 232 }
 233
 234 static void
 235 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 236                               struct rte_cryptodev_sym_session *sess)
 237 {
 238         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 239         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 240                                                                 dev->driver_id);
 241
 242         if (unlikely(spriv == NULL)) {
 243                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 244                 return;
 245         }
 246         mlx5_crypto_dek_destroy(priv, spriv->dek);
 247         set_sym_session_private_data(sess, dev->driver_id, NULL);
 248         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 249         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 250 }
 251
 252 static int
 253 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 254 {
 255         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 256
 257         if (qp->qp_obj != NULL)
 258                 claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
 259         if (qp->umem_obj != NULL)
 260                 claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 261         if (qp->umem_buf != NULL)
 262                 rte_free(qp->umem_buf);
 263         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 264         mlx5_devx_cq_destroy(&qp->cq_obj);
 265         rte_free(qp);
 266         dev->data->queue_pairs[qp_id] = NULL;
 267         return 0;
 268 }
 269
 270 static int
 271 mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 272 {
 273         /*
 274          * In Order to configure self loopback, when calling these functions the
 275          * remote QP id that is used is the id of the same QP.
 276          */
 277         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
 278                                           qp->qp_obj->id)) {
 279                 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
 280                         rte_errno);
 281                 return -1;
 282         }
 283         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
 284                                           qp->qp_obj->id)) {
 285                 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
 286                         rte_errno);
 287                 return -1;
 288         }
 289         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
 290                                           qp->qp_obj->id)) {
 291                 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
 292                         rte_errno);
 293                 return -1;
 294         }
 295         return 0;
 296 }
 297
 298 static int
 299 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 300                              const struct rte_cryptodev_qp_conf *qp_conf,
 301                              int socket_id)
 302 {
 303         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 304         struct mlx5_devx_qp_attr attr = {0};
 305         struct mlx5_crypto_qp *qp;
 306         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 307         uint32_t umem_size = RTE_BIT32(log_nb_desc) *
 308                               MLX5_CRYPTO_WQE_SET_SIZE +
 309                               sizeof(*qp->db_rec) * 2;
 310         uint32_t alloc_size = sizeof(*qp);
 311         struct mlx5_devx_cq_attr cq_attr = {
 312                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
 313         };
 314
 315         if (dev->data->queue_pairs[qp_id] != NULL)
 316                 mlx5_crypto_queue_pair_release(dev, qp_id);
 317         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 318         alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
 319         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 320                                 socket_id);
 321         if (qp == NULL) {
 322                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 323                 rte_errno = ENOMEM;
 324                 return -rte_errno;
 325         }
 326         if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
 327                                 &cq_attr, socket_id) != 0) {
 328                 DRV_LOG(ERR, "Failed to create CQ.");
 329                 goto error;
 330         }
 331         qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
 332         if (qp->umem_buf == NULL) {
 333                 DRV_LOG(ERR, "Failed to allocate QP umem.");
 334                 rte_errno = ENOMEM;
 335                 goto error;
 336         }
 337         qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 338                                                (void *)(uintptr_t)qp->umem_buf,
 339                                                umem_size,
 340                                                IBV_ACCESS_LOCAL_WRITE);
 341         if (qp->umem_obj == NULL) {
 342                 DRV_LOG(ERR, "Failed to register QP umem.");
 343                 goto error;
 344         }
 345         if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
 346                                priv->dev_config.socket_id) != 0) {
 347                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 348                         (uint32_t)qp_id);
 349                 rte_errno = ENOMEM;
 350                 goto error;
 351         }
 352         qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 353         attr.pd = priv->pdn;
 354         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 355         attr.cqn = qp->cq_obj.cq->id;
 356         attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
 357         attr.rq_size =  0;
 358         attr.sq_size = RTE_BIT32(log_nb_desc);
 359         attr.dbr_umem_valid = 1;
 360         attr.wq_umem_id = qp->umem_obj->umem_id;
 361         attr.wq_umem_offset = 0;
 362         attr.dbr_umem_id = qp->umem_obj->umem_id;
 363         attr.dbr_address = RTE_BIT64(log_nb_desc) *
 364                            MLX5_CRYPTO_WQE_SET_SIZE;
 365         qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 366         if (qp->qp_obj == NULL) {
 367                 DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
 368                 goto error;
 369         }
 370         qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 371         if (mlx5_crypto_qp2rts(qp))
 372                 goto error;
 373         qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
 374                                                            RTE_CACHE_LINE_SIZE);
 375         dev->data->queue_pairs[qp_id] = qp;
 376         return 0;
 377 error:
 378         mlx5_crypto_queue_pair_release(dev, qp_id);
 379         return -1;
 380 }
 381
 382 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 383         .dev_configure                  = mlx5_crypto_dev_configure,
 384         .dev_start                      = mlx5_crypto_dev_start,
 385         .dev_stop                       = mlx5_crypto_dev_stop,
 386         .dev_close                      = mlx5_crypto_dev_close,
 387         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 388         .stats_get                      = NULL,
 389         .stats_reset                    = NULL,
 390         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 391         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 392         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 393         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 394         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 395         .sym_get_raw_dp_ctx_size        = NULL,
 396         .sym_configure_raw_dp_ctx       = NULL,
 397 };
 398
 399 static void
 400 mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
 401 {
 402         if (priv->pd != NULL) {
 403                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 404                 priv->pd = NULL;
 405         }
 406         if (priv->uar != NULL) {
 407                 mlx5_glue->devx_free_uar(priv->uar);
 408                 priv->uar = NULL;
 409         }
 410 }
 411
 412 static int
 413 mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
 414 {
 415 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 416         struct mlx5dv_obj obj;
 417         struct mlx5dv_pd pd_info;
 418         int ret;
 419
 420         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 421         if (priv->pd == NULL) {
 422                 DRV_LOG(ERR, "Failed to allocate PD.");
 423                 return errno ? -errno : -ENOMEM;
 424         }
 425         obj.pd.in = priv->pd;
 426         obj.pd.out = &pd_info;
 427         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 428         if (ret != 0) {
 429                 DRV_LOG(ERR, "Fail to get PD object info.");
 430                 mlx5_glue->dealloc_pd(priv->pd);
 431                 priv->pd = NULL;
 432                 return -errno;
 433         }
 434         priv->pdn = pd_info.pdn;
 435         return 0;
 436 #else
 437         (void)priv;
 438         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 439         return -ENOTSUP;
 440 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 441 }
 442
 443 static int
 444 mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 445 {
 446         if (mlx5_crypto_pd_create(priv) != 0)
 447                 return -1;
 448         priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
 449         if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
 450             NULL) {
 451                 rte_errno = errno;
 452                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 453                 DRV_LOG(ERR, "Failed to allocate UAR.");
 454                 return -1;
 455         }
 456         return 0;
 457 }
 458
 459
 460 static int
 461 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 462 {
 463         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 464         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 465         unsigned long tmp;
 466         FILE *file;
 467         int ret;
 468         int i;
 469
 470         if (strcmp(key, "class") == 0)
 471                 return 0;
 472         if (strcmp(key, "wcs_file") == 0) {
 473                 file = fopen(val, "rb");
 474                 if (file == NULL) {
 475                         rte_errno = ENOTSUP;
 476                         return -rte_errno;
 477                 }
 478                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 479                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 480                         if (ret <= 0) {
 481                                 fclose(file);
 482                                 DRV_LOG(ERR,
 483                                         "Failed to read credential from file.");
 484                                 rte_errno = EINVAL;
 485                                 return -rte_errno;
 486                         }
 487                 }
 488                 fclose(file);
 489                 devarg_prms->login_devarg = true;
 490                 return 0;
 491         }
 492         errno = 0;
 493         tmp = strtoul(val, NULL, 0);
 494         if (errno) {
 495                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 496                 return -errno;
 497         }
 498         if (strcmp(key, "max_segs_num") == 0) {
 499                 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
 500                         DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
 501                                 " be less than %d.",
 502                                 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
 503                         rte_errno = EINVAL;
 504                         return -rte_errno;
 505                 }
 506                 devarg_prms->max_segs_num = (uint32_t)tmp;
 507         } else if (strcmp(key, "import_kek_id") == 0) {
 508                 attr->session_import_kek_ptr = (uint32_t)tmp;
 509         } else if (strcmp(key, "credential_id") == 0) {
 510                 attr->credential_pointer = (uint32_t)tmp;
 511         } else if (strcmp(key, "keytag") == 0) {
 512                 devarg_prms->keytag = tmp;
 513         } else {
 514                 DRV_LOG(WARNING, "Invalid key %s.", key);
 515         }
 516         return 0;
 517 }
 518
 519 static int
 520 mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 521                           struct mlx5_crypto_devarg_params *devarg_prms)
 522 {
 523         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 524         struct rte_kvargs *kvlist;
 525
 526         /* Default values. */
 527         attr->credential_pointer = 0;
 528         attr->session_import_kek_ptr = 0;
 529         devarg_prms->keytag = 0;
 530         devarg_prms->max_segs_num = 8;
 531         if (devargs == NULL) {
 532                 DRV_LOG(ERR,
 533         "No login devargs in order to enable crypto operations in the device.");
 534                 rte_errno = EINVAL;
 535                 return -1;
 536         }
 537         kvlist = rte_kvargs_parse(devargs->args, NULL);
 538         if (kvlist == NULL) {
 539                 DRV_LOG(ERR, "Failed to parse devargs.");
 540                 rte_errno = EINVAL;
 541                 return -1;
 542         }
 543         if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
 544                            devarg_prms) != 0) {
 545                 DRV_LOG(ERR, "Devargs handler function Failed.");
 546                 rte_kvargs_free(kvlist);
 547                 rte_errno = EINVAL;
 548                 return -1;
 549         }
 550         rte_kvargs_free(kvlist);
 551         if (devarg_prms->login_devarg == false) {
 552                 DRV_LOG(ERR,
 553         "No login credential devarg in order to enable crypto operations "
 554         "in the device.");
 555                 rte_errno = EINVAL;
 556                 return -1;
 557         }
 558         return 0;
 559 }
 560
 561 /**
 562  * Callback for memory event.
 563  *
 564  * @param event_type
 565  *   Memory event type.
 566  * @param addr
 567  *   Address of memory.
 568  * @param len
 569  *   Size of memory.
 570  */
 571 static void
 572 mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 573                             size_t len, void *arg __rte_unused)
 574 {
 575         struct mlx5_crypto_priv *priv;
 576
 577         /* Must be called from the primary process. */
 578         MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
 579         switch (event_type) {
 580         case RTE_MEM_EVENT_FREE:
 581                 pthread_mutex_lock(&priv_list_lock);
 582                 /* Iterate all the existing mlx5 devices. */
 583                 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 584                         mlx5_free_mr_by_addr(&priv->mr_scache,
 585                                              priv->ctx->device->name,
 586                                              addr, len);
 587                 pthread_mutex_unlock(&priv_list_lock);
 588                 break;
 589         case RTE_MEM_EVENT_ALLOC:
 590         default:
 591                 break;
 592         }
 593 }
 594
 595 /**
 596  * DPDK callback to register a PCI device.
 597  *
 598  * This function spawns crypto device out of a given PCI device.
 599  *
 600  * @param[in] pci_drv
 601  *   PCI driver structure (mlx5_crypto_driver).
 602  * @param[in] pci_dev
 603  *   PCI device information.
 604  *
 605  * @return
 606  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 607  *   and rte_errno is set.
 608  */
 609 static int
 610 mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 611                         struct rte_pci_device *pci_dev)
 612 {
 613         struct ibv_device *ibv;
 614         struct rte_cryptodev *crypto_dev;
 615         struct ibv_context *ctx;
 616         struct mlx5_devx_obj *login;
 617         struct mlx5_crypto_priv *priv;
 618         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 619         struct mlx5_hca_attr attr = { 0 };
 620         struct rte_cryptodev_pmd_init_params init_params = {
 621                 .name = "",
 622                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 623                 .socket_id = pci_dev->device.numa_node,
 624                 .max_nb_queue_pairs =
 625                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 626         };
 627         uint16_t rdmw_wqe_size;
 628         int ret;
 629
 630         RTE_SET_USED(pci_drv);
 631         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 632                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 633                 rte_errno = ENOTSUP;
 634                 return -rte_errno;
 635         }
 636         ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
 637         if (ibv == NULL) {
 638                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 639                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 640                         pci_dev->addr.bus, pci_dev->addr.devid,
 641                         pci_dev->addr.function);
 642                 return -rte_errno;
 643         }
 644         DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
 645         ctx = mlx5_glue->dv_open_device(ibv);
 646         if (ctx == NULL) {
 647                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 648                 rte_errno = ENODEV;
 649                 return -rte_errno;
 650         }
 651         if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
 652             attr.crypto == 0 || attr.aes_xts == 0) {
 653                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 654                         "operations, maybe old FW/OFED version?");
 655                 claim_zero(mlx5_glue->close_device(ctx));
 656                 rte_errno = ENOTSUP;
 657                 return -ENOTSUP;
 658         }
 659         ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
 660         if (ret) {
 661                 DRV_LOG(ERR, "Failed to parse devargs.");
 662                 return -rte_errno;
 663         }
 664         login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
 665                                                       &devarg_prms.login_attr);
 666         if (login == NULL) {
 667                 DRV_LOG(ERR, "Failed to configure login.");
 668                 return -rte_errno;
 669         }
 670         crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 671                                         &init_params);
 672         if (crypto_dev == NULL) {
 673                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
 674                 claim_zero(mlx5_glue->close_device(ctx));
 675                 return -ENODEV;
 676         }
 677         DRV_LOG(INFO,
 678                 "Crypto device %s was created successfully.", ibv->name);
 679         crypto_dev->dev_ops = &mlx5_crypto_ops;
 680         crypto_dev->dequeue_burst = NULL;
 681         crypto_dev->enqueue_burst = NULL;
 682         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 683         crypto_dev->driver_id = mlx5_crypto_driver_id;
 684         priv = crypto_dev->data->dev_private;
 685         priv->ctx = ctx;
 686         priv->login_obj = login;
 687         priv->pci_dev = pci_dev;
 688         priv->crypto_dev = crypto_dev;
 689         if (mlx5_crypto_hw_global_prepare(priv) != 0) {
 690                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 691                 claim_zero(mlx5_glue->close_device(priv->ctx));
 692                 return -1;
 693         }
 694         if (mlx5_mr_btree_init(&priv->mr_scache.cache,
 695                              MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
 696                 DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
 697                 mlx5_crypto_hw_global_release(priv);
 698                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 699                 claim_zero(mlx5_glue->close_device(priv->ctx));
 700                 rte_errno = ENOMEM;
 701                 return -rte_errno;
 702         }
 703         priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 704         priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 705         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 706         priv->max_segs_num = devarg_prms.max_segs_num;
 707         priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
 708                              sizeof(struct mlx5_umr_wqe) +
 709                              RTE_ALIGN(priv->max_segs_num, 4) *
 710                              sizeof(struct mlx5_wqe_dseg);
 711         rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
 712                               sizeof(struct mlx5_wqe_dseg) *
 713                               (priv->max_segs_num <= 2 ? 2 : 2 +
 714                                RTE_ALIGN(priv->max_segs_num - 2, 4));
 715         priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
 716         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
 717         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 718         /* Register callback function for global shared MR cache management. */
 719         if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 720                 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
 721                                                 mlx5_crypto_mr_mem_event_cb,
 722                                                 NULL);
 723         pthread_mutex_lock(&priv_list_lock);
 724         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 725         pthread_mutex_unlock(&priv_list_lock);
 726         return 0;
 727 }
 728
 729 static int
 730 mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 731 {
 732         struct mlx5_crypto_priv *priv = NULL;
 733
 734         pthread_mutex_lock(&priv_list_lock);
 735         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 736                 if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
 737                         break;
 738         if (priv)
 739                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 740         pthread_mutex_unlock(&priv_list_lock);
 741         if (priv) {
 742                 if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 743                         rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
 744                                                           NULL);
 745                 mlx5_mr_release_cache(&priv->mr_scache);
 746                 mlx5_crypto_hw_global_release(priv);
 747                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 748                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 749                 claim_zero(mlx5_glue->close_device(priv->ctx));
 750         }
 751         return 0;
 752 }
 753
 754 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 755                 {
 756                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 757                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 758                 },
 759                 {
 760                         .vendor_id = 0
 761                 }
 762 };
 763
 764 static struct mlx5_pci_driver mlx5_crypto_driver = {
 765         .driver_class = MLX5_CLASS_CRYPTO,
 766         .pci_driver = {
 767                 .driver = {
 768                         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
 769                 },
 770                 .id_table = mlx5_crypto_pci_id_map,
 771                 .probe = mlx5_crypto_pci_probe,
 772                 .remove = mlx5_crypto_pci_remove,
 773                 .drv_flags = 0,
 774         },
 775 };
 776
 777 RTE_INIT(rte_mlx5_crypto_init)
 778 {
 779         mlx5_common_init();
 780         if (mlx5_glue != NULL)
 781                 mlx5_pci_driver_register(&mlx5_crypto_driver);
 782 }
 783
 784 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
 785                                mlx5_crypto_driver_id);
 786
 787 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
 788 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
 789 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
 790 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");