drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_errno.h>
   8 #include <rte_log.h>
   9 #include <rte_pci.h>
  10 #include <rte_memory.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_common.h>
  14 #include <mlx5_common_pci.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 1024
  24 #define MLX5_CRYPTO_MAX_SEGS 56
  25
  26 #define MLX5_CRYPTO_FEATURE_FLAGS \
  27         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  28          RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
  29          RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
  30          RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
  31          RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
  32          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  33          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  34
  35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  36                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  37 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  38
  39 int mlx5_crypto_logtype;
  40
  41 uint8_t mlx5_crypto_driver_id;
  42
  43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  44         {               /* AES XTS */
  45                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  46                 {.sym = {
  47                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  48                         {.cipher = {
  49                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  50                                 .block_size = 16,
  51                                 .key_size = {
  52                                         .min = 32,
  53                                         .max = 64,
  54                                         .increment = 32
  55                                 },
  56                                 .iv_size = {
  57                                         .min = 16,
  58                                         .max = 16,
  59                                         .increment = 0
  60                                 },
  61                                 .dataunit_set =
  62                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  63                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
  64                         }, }
  65                 }, }
  66         },
  67 };
  68
  69 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  70
  71 static const struct rte_driver mlx5_drv = {
  72         .name = mlx5_crypto_drv_name,
  73         .alias = mlx5_crypto_drv_name
  74 };
  75
  76 static struct cryptodev_driver mlx5_cryptodev_driver;
  77
  78 struct mlx5_crypto_session {
  79         uint32_t bs_bpt_eo_es;
  80         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  81          * saved in big endian format.
  82          */
  83         uint32_t bsp_res;
  84         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  85          * endian format.
  86          */
  87         uint32_t iv_offset:16;
  88         /**< Starting point for Initialisation Vector. */
  89         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  90         uint32_t dek_id; /**< DEK ID */
  91 } __rte_packed;
  92
  93 static void
  94 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  95                           struct rte_cryptodev_info *dev_info)
  96 {
  97         RTE_SET_USED(dev);
  98         if (dev_info != NULL) {
  99                 dev_info->driver_id = mlx5_crypto_driver_id;
 100                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 101                 dev_info->capabilities = mlx5_crypto_caps;
 102                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 103                 dev_info->min_mbuf_headroom_req = 0;
 104                 dev_info->min_mbuf_tailroom_req = 0;
 105                 dev_info->sym.max_nb_sessions = 0;
 106                 /*
 107                  * If 0, the device does not have any limitation in number of
 108                  * sessions that can be used.
 109                  */
 110         }
 111 }
 112
 113 static int
 114 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 115                           struct rte_cryptodev_config *config)
 116 {
 117         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 118
 119         if (config == NULL) {
 120                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 121                 return -EINVAL;
 122         }
 123         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 124                 DRV_LOG(ERR,
 125                         "Disabled symmetric crypto feature is not supported.");
 126                 return -ENOTSUP;
 127         }
 128         if (mlx5_crypto_dek_setup(priv) != 0) {
 129                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 130                 return -ENOMEM;
 131         }
 132         priv->dev_config = *config;
 133         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 134         return 0;
 135 }
 136
 137 static void
 138 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 139 {
 140         RTE_SET_USED(dev);
 141 }
 142
 143 static int
 144 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 145 {
 146         RTE_SET_USED(dev);
 147         return 0;
 148 }
 149
 150 static int
 151 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 152 {
 153         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 154
 155         mlx5_crypto_dek_unset(priv);
 156         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 157         return 0;
 158 }
 159
 160 static unsigned int
 161 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 162 {
 163         return sizeof(struct mlx5_crypto_session);
 164 }
 165
 166 static int
 167 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 168                                   struct rte_crypto_sym_xform *xform,
 169                                   struct rte_cryptodev_sym_session *session,
 170                                   struct rte_mempool *mp)
 171 {
 172         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 173         struct mlx5_crypto_session *sess_private_data;
 174         struct rte_crypto_cipher_xform *cipher;
 175         uint8_t encryption_order;
 176         int ret;
 177
 178         if (unlikely(xform->next != NULL)) {
 179                 DRV_LOG(ERR, "Xform next is not supported.");
 180                 return -ENOTSUP;
 181         }
 182         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 183                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 184                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 185                 return -ENOTSUP;
 186         }
 187         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 188         if (ret != 0) {
 189                 DRV_LOG(ERR,
 190                         "Failed to get session %p private data from mempool.",
 191                         sess_private_data);
 192                 return -ENOMEM;
 193         }
 194         cipher = &xform->cipher;
 195         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 196         if (sess_private_data->dek == NULL) {
 197                 rte_mempool_put(mp, sess_private_data);
 198                 DRV_LOG(ERR, "Failed to prepare dek.");
 199                 return -ENOMEM;
 200         }
 201         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 202                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 203         else
 204                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 205         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 206                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 207                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 208                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 209                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 210         switch (xform->cipher.dataunit_len) {
 211         case 0:
 212                 sess_private_data->bsp_res = 0;
 213                 break;
 214         case 512:
 215                 sess_private_data->bsp_res = rte_cpu_to_be_32
 216                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 217                                              MLX5_BLOCK_SIZE_OFFSET);
 218                 break;
 219         case 4096:
 220                 sess_private_data->bsp_res = rte_cpu_to_be_32
 221                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 222                                              MLX5_BLOCK_SIZE_OFFSET);
 223                 break;
 224         default:
 225                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 226                 return -ENOTSUP;
 227         }
 228         sess_private_data->iv_offset = cipher->iv.offset;
 229         sess_private_data->dek_id =
 230                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 231                                          0xffffff);
 232         set_sym_session_private_data(session, dev->driver_id,
 233                                      sess_private_data);
 234         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 235         return 0;
 236 }
 237
 238 static void
 239 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 240                               struct rte_cryptodev_sym_session *sess)
 241 {
 242         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 243         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 244                                                                 dev->driver_id);
 245
 246         if (unlikely(spriv == NULL)) {
 247                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 248                 return;
 249         }
 250         mlx5_crypto_dek_destroy(priv, spriv->dek);
 251         set_sym_session_private_data(sess, dev->driver_id, NULL);
 252         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 253         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 254 }
 255
 256 static int
 257 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 258 {
 259         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 260
 261         if (qp->qp_obj != NULL)
 262                 claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
 263         if (qp->umem_obj != NULL)
 264                 claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 265         if (qp->umem_buf != NULL)
 266                 rte_free(qp->umem_buf);
 267         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 268         mlx5_devx_cq_destroy(&qp->cq_obj);
 269         rte_free(qp);
 270         dev->data->queue_pairs[qp_id] = NULL;
 271         return 0;
 272 }
 273
 274 static int
 275 mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 276 {
 277         /*
 278          * In Order to configure self loopback, when calling these functions the
 279          * remote QP id that is used is the id of the same QP.
 280          */
 281         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
 282                                           qp->qp_obj->id)) {
 283                 DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
 284                         rte_errno);
 285                 return -1;
 286         }
 287         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
 288                                           qp->qp_obj->id)) {
 289                 DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
 290                         rte_errno);
 291                 return -1;
 292         }
 293         if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
 294                                           qp->qp_obj->id)) {
 295                 DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
 296                         rte_errno);
 297                 return -1;
 298         }
 299         return 0;
 300 }
 301
 302 static __rte_noinline uint32_t
 303 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 304 {
 305         uint32_t bl = op->sym->cipher.data.length;
 306
 307         switch (bl) {
 308         case (1 << 20):
 309                 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
 310         case (1 << 12):
 311                 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
 312                                 MLX5_BLOCK_SIZE_OFFSET);
 313         case (1 << 9):
 314                 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
 315         default:
 316                 DRV_LOG(ERR, "Unknown block size: %u.", bl);
 317                 return UINT32_MAX;
 318         }
 319 }
 320
 321 /**
 322  * Query LKey from a packet buffer for QP. If not found, add the mempool.
 323  *
 324  * @param priv
 325  *   Pointer to the priv object.
 326  * @param addr
 327  *   Search key.
 328  * @param mr_ctrl
 329  *   Pointer to per-queue MR control structure.
 330  * @param ol_flags
 331  *   Mbuf offload features.
 332  *
 333  * @return
 334  *   Searched LKey on success, UINT32_MAX on no match.
 335  */
 336 static __rte_always_inline uint32_t
 337 mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
 338                     struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
 339 {
 340         uint32_t lkey;
 341
 342         /* Check generation bit to see if there's any change on existing MRs. */
 343         if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
 344                 mlx5_mr_flush_local_cache(mr_ctrl);
 345         /* Linear search on MR cache array. */
 346         lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
 347                                    MLX5_MR_CACHE_N, addr);
 348         if (likely(lkey != UINT32_MAX))
 349                 return lkey;
 350         /* Take slower bottom-half on miss. */
 351         return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,
 352                                   !!(ol_flags & EXT_ATTACHED_MBUF));
 353 }
 354
 355 static __rte_always_inline uint32_t
 356 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 357                       struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
 358                       uint32_t offset, uint32_t *remain)
 359 {
 360         uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 361         uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 362
 363         if (data_len > *remain)
 364                 data_len = *remain;
 365         *remain -= data_len;
 366         klm->bcount = rte_cpu_to_be_32(data_len);
 367         klm->pbuf = rte_cpu_to_be_64(addr);
 368         klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
 369                                         mbuf->ol_flags);
 370         return klm->lkey;
 371
 372 }
 373
 374 static __rte_always_inline uint32_t
 375 mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 376                      struct rte_crypto_op *op, struct rte_mbuf *mbuf,
 377                      struct mlx5_wqe_dseg *klm)
 378 {
 379         uint32_t remain_len = op->sym->cipher.data.length;
 380         uint32_t nb_segs = mbuf->nb_segs;
 381         uint32_t klm_n = 1u;
 382
 383         /* First mbuf needs to take the cipher offset. */
 384         if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
 385                      op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 386                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 387                 return 0;
 388         }
 389         while (remain_len) {
 390                 nb_segs--;
 391                 mbuf = mbuf->next;
 392                 if (unlikely(mbuf == NULL || nb_segs == 0)) {
 393                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 394                         return 0;
 395                 }
 396                 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
 397                                                  &remain_len) == UINT32_MAX)) {
 398                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 399                         return 0;
 400                 }
 401                 klm_n++;
 402         }
 403         return klm_n;
 404 }
 405
 406 static __rte_always_inline int
 407 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 408                          struct mlx5_crypto_qp *qp,
 409                          struct rte_crypto_op *op,
 410                          struct mlx5_umr_wqe *umr)
 411 {
 412         struct mlx5_crypto_session *sess = get_sym_session_private_data
 413                                 (op->sym->session, mlx5_crypto_driver_id);
 414         struct mlx5_wqe_cseg *cseg = &umr->ctr;
 415         struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
 416         struct mlx5_wqe_dseg *klms = &umr->kseg[0];
 417         struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
 418                                       RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
 419         uint32_t ds;
 420         bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 421         /* Set UMR WQE. */
 422         uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
 423                                    ipl ? op->sym->m_src : op->sym->m_dst, klms);
 424
 425         if (unlikely(klm_n == 0))
 426                 return 0;
 427         bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
 428         if (unlikely(!sess->bsp_res)) {
 429                 bsf->bsp_res = mlx5_crypto_get_block_size(op);
 430                 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
 431                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 432                         return 0;
 433                 }
 434         } else {
 435                 bsf->bsp_res = sess->bsp_res;
 436         }
 437         bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
 438         memcpy(bsf->xts_initial_tweak,
 439                rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
 440         bsf->res_dp = sess->dek_id;
 441         mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
 442         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
 443         qp->db_pi += priv->umr_wqe_stride;
 444         /* Set RDMA_WRITE WQE. */
 445         cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 446         klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 447         if (!ipl) {
 448                 klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
 449                                              klms);
 450                 if (unlikely(klm_n == 0))
 451                         return 0;
 452         } else {
 453                 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
 454         }
 455         ds = 2 + klm_n;
 456         cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
 457         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 458                                                         MLX5_OPCODE_RDMA_WRITE);
 459         ds = RTE_ALIGN(ds, 4);
 460         qp->db_pi += ds >> 2;
 461         /* Set NOP WQE if needed. */
 462         if (priv->max_rdmar_ds > ds) {
 463                 cseg += ds;
 464                 ds = priv->max_rdmar_ds - ds;
 465                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
 466                 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 467                                                                MLX5_OPCODE_NOP);
 468                 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
 469         }
 470         qp->wqe = (uint8_t *)cseg;
 471         return 1;
 472 }
 473
 474 static __rte_always_inline void
 475 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
 476 {
 477 #ifdef RTE_ARCH_64
 478         *priv->uar_addr = val;
 479 #else /* !RTE_ARCH_64 */
 480         rte_spinlock_lock(&priv->uar32_sl);
 481         *(volatile uint32_t *)priv->uar_addr = val;
 482         rte_io_wmb();
 483         *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
 484         rte_spinlock_unlock(&priv->uar32_sl);
 485 #endif
 486 }
 487
 488 static uint16_t
 489 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 490                           uint16_t nb_ops)
 491 {
 492         struct mlx5_crypto_qp *qp = queue_pair;
 493         struct mlx5_crypto_priv *priv = qp->priv;
 494         struct mlx5_umr_wqe *umr;
 495         struct rte_crypto_op *op;
 496         uint16_t mask = qp->entries_n - 1;
 497         uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
 498
 499         if (remain < nb_ops)
 500                 nb_ops = remain;
 501         else
 502                 remain = nb_ops;
 503         if (unlikely(remain == 0))
 504                 return 0;
 505         do {
 506                 op = *ops++;
 507                 umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
 508                 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
 509                         if (remain != nb_ops)
 510                                 break;
 511                         return 0;
 512                 }
 513                 qp->ops[qp->pi] = op;
 514                 qp->pi = (qp->pi + 1) & mask;
 515         } while (--remain);
 516         rte_io_wmb();
 517         qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 518         rte_wmb();
 519         mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
 520         rte_wmb();
 521         return nb_ops;
 522 }
 523
 524 static __rte_noinline void
 525 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 526 {
 527         const uint32_t idx = qp->ci & (qp->entries_n - 1);
 528         volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
 529                                                         &qp->cq_obj.cqes[idx];
 530
 531         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 532         DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 533 }
 534
 535 static uint16_t
 536 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 537                           uint16_t nb_ops)
 538 {
 539         struct mlx5_crypto_qp *qp = queue_pair;
 540         volatile struct mlx5_cqe *restrict cqe;
 541         struct rte_crypto_op *restrict op;
 542         const unsigned int cq_size = qp->entries_n;
 543         const unsigned int mask = cq_size - 1;
 544         uint32_t idx;
 545         uint32_t next_idx = qp->ci & mask;
 546         const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
 547         uint16_t i = 0;
 548         int ret;
 549
 550         if (unlikely(max == 0))
 551                 return 0;
 552         do {
 553                 idx = next_idx;
 554                 next_idx = (qp->ci + 1) & mask;
 555                 op = qp->ops[idx];
 556                 cqe = &qp->cq_obj.cqes[idx];
 557                 ret = check_cqe(cqe, cq_size, qp->ci);
 558                 rte_io_rmb();
 559                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 560                         if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
 561                                 mlx5_crypto_cqe_err_handle(qp, op);
 562                         break;
 563                 }
 564                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 565                 ops[i++] = op;
 566                 qp->ci++;
 567         } while (i < max);
 568         if (likely(i != 0)) {
 569                 rte_io_wmb();
 570                 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
 571         }
 572         return i;
 573 }
 574
 575 static void
 576 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 577 {
 578         uint32_t i;
 579
 580         for (i = 0 ; i < qp->entries_n; i++) {
 581                 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
 582                                                          priv->wqe_set_size);
 583                 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
 584                                                                      (cseg + 1);
 585                 struct mlx5_wqe_umr_bsf_seg *bsf =
 586                         (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
 587                                                        priv->umr_wqe_size)) - 1;
 588                 struct mlx5_wqe_rseg *rseg;
 589
 590                 /* Init UMR WQE. */
 591                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
 592                                          (priv->umr_wqe_size / MLX5_WSEG_SIZE));
 593                 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 594                                        MLX5_COMP_MODE_OFFSET);
 595                 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
 596                 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
 597                 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
 598                 ucseg->ko_to_bs = rte_cpu_to_be_32
 599                         ((RTE_ALIGN(priv->max_segs_num, 4u) <<
 600                          MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
 601                 bsf->keytag = priv->keytag;
 602                 /* Init RDMA WRITE WQE. */
 603                 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 604                 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
 605                                       MLX5_COMP_MODE_OFFSET) |
 606                                       MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
 607                 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
 608                 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
 609         }
 610 }
 611
 612 static int
 613 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
 614                                   struct mlx5_crypto_qp *qp)
 615 {
 616         struct mlx5_umr_wqe *umr;
 617         uint32_t i;
 618         struct mlx5_devx_mkey_attr attr = {
 619                 .pd = priv->pdn,
 620                 .umr_en = 1,
 621                 .crypto_en = 1,
 622                 .set_remote_rw = 1,
 623                 .klm_num = RTE_ALIGN(priv->max_segs_num, 4),
 624         };
 625
 626         for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
 627            i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
 628                 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
 629                 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
 630                 if (!qp->mkey[i]) {
 631                         DRV_LOG(ERR, "Failed to allocate indirect mkey.");
 632                         return -1;
 633                 }
 634         }
 635         return 0;
 636 }
 637
 638 static int
 639 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 640                              const struct rte_cryptodev_qp_conf *qp_conf,
 641                              int socket_id)
 642 {
 643         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 644         struct mlx5_devx_qp_attr attr = {0};
 645         struct mlx5_crypto_qp *qp;
 646         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 647         uint32_t umem_size = RTE_BIT32(log_nb_desc) *
 648                               priv->wqe_set_size +
 649                               sizeof(*qp->db_rec) * 2;
 650         uint32_t alloc_size = sizeof(*qp);
 651         struct mlx5_devx_cq_attr cq_attr = {
 652                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
 653         };
 654
 655         if (dev->data->queue_pairs[qp_id] != NULL)
 656                 mlx5_crypto_queue_pair_release(dev, qp_id);
 657         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 658         alloc_size += (sizeof(struct rte_crypto_op *) +
 659                        sizeof(struct mlx5_devx_obj *)) *
 660                        RTE_BIT32(log_nb_desc);
 661         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 662                                 socket_id);
 663         if (qp == NULL) {
 664                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 665                 rte_errno = ENOMEM;
 666                 return -rte_errno;
 667         }
 668         if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
 669                                 &cq_attr, socket_id) != 0) {
 670                 DRV_LOG(ERR, "Failed to create CQ.");
 671                 goto error;
 672         }
 673         qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
 674         if (qp->umem_buf == NULL) {
 675                 DRV_LOG(ERR, "Failed to allocate QP umem.");
 676                 rte_errno = ENOMEM;
 677                 goto error;
 678         }
 679         qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 680                                                (void *)(uintptr_t)qp->umem_buf,
 681                                                umem_size,
 682                                                IBV_ACCESS_LOCAL_WRITE);
 683         if (qp->umem_obj == NULL) {
 684                 DRV_LOG(ERR, "Failed to register QP umem.");
 685                 goto error;
 686         }
 687         if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
 688                                priv->dev_config.socket_id) != 0) {
 689                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 690                         (uint32_t)qp_id);
 691                 rte_errno = ENOMEM;
 692                 goto error;
 693         }
 694         qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 695         attr.pd = priv->pdn;
 696         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 697         attr.cqn = qp->cq_obj.cq->id;
 698         attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
 699         attr.rq_size =  0;
 700         attr.sq_size = RTE_BIT32(log_nb_desc);
 701         attr.dbr_umem_valid = 1;
 702         attr.wq_umem_id = qp->umem_obj->umem_id;
 703         attr.wq_umem_offset = 0;
 704         attr.dbr_umem_id = qp->umem_obj->umem_id;
 705         attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 706         qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 707         if (qp->qp_obj == NULL) {
 708                 DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
 709                 goto error;
 710         }
 711         qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 712         if (mlx5_crypto_qp2rts(qp))
 713                 goto error;
 714         qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 715                                                            RTE_CACHE_LINE_SIZE);
 716         qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
 717         qp->entries_n = 1 << log_nb_desc;
 718         if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
 719                 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
 720                 rte_errno = ENOMEM;
 721                 goto error;
 722         }
 723         mlx5_crypto_qp_init(priv, qp);
 724         qp->priv = priv;
 725         dev->data->queue_pairs[qp_id] = qp;
 726         return 0;
 727 error:
 728         mlx5_crypto_queue_pair_release(dev, qp_id);
 729         return -1;
 730 }
 731
 732 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 733         .dev_configure                  = mlx5_crypto_dev_configure,
 734         .dev_start                      = mlx5_crypto_dev_start,
 735         .dev_stop                       = mlx5_crypto_dev_stop,
 736         .dev_close                      = mlx5_crypto_dev_close,
 737         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 738         .stats_get                      = NULL,
 739         .stats_reset                    = NULL,
 740         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 741         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 742         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 743         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 744         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 745         .sym_get_raw_dp_ctx_size        = NULL,
 746         .sym_configure_raw_dp_ctx       = NULL,
 747 };
 748
 749 static void
 750 mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
 751 {
 752         if (priv->pd != NULL) {
 753                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 754                 priv->pd = NULL;
 755         }
 756         if (priv->uar != NULL) {
 757                 mlx5_glue->devx_free_uar(priv->uar);
 758                 priv->uar = NULL;
 759         }
 760 }
 761
 762 static int
 763 mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
 764 {
 765 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 766         struct mlx5dv_obj obj;
 767         struct mlx5dv_pd pd_info;
 768         int ret;
 769
 770         priv->pd = mlx5_glue->alloc_pd(priv->ctx);
 771         if (priv->pd == NULL) {
 772                 DRV_LOG(ERR, "Failed to allocate PD.");
 773                 return errno ? -errno : -ENOMEM;
 774         }
 775         obj.pd.in = priv->pd;
 776         obj.pd.out = &pd_info;
 777         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
 778         if (ret != 0) {
 779                 DRV_LOG(ERR, "Fail to get PD object info.");
 780                 mlx5_glue->dealloc_pd(priv->pd);
 781                 priv->pd = NULL;
 782                 return -errno;
 783         }
 784         priv->pdn = pd_info.pdn;
 785         return 0;
 786 #else
 787         (void)priv;
 788         DRV_LOG(ERR, "Cannot get pdn - no DV support.");
 789         return -ENOTSUP;
 790 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
 791 }
 792
 793 static int
 794 mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 795 {
 796         if (mlx5_crypto_pd_create(priv) != 0)
 797                 return -1;
 798         priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
 799         if (priv->uar)
 800                 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
 801         if (priv->uar == NULL || priv->uar_addr == NULL) {
 802                 rte_errno = errno;
 803                 claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 804                 DRV_LOG(ERR, "Failed to allocate UAR.");
 805                 return -1;
 806         }
 807         return 0;
 808 }
 809
 810
 811 static int
 812 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 813 {
 814         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 815         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 816         unsigned long tmp;
 817         FILE *file;
 818         int ret;
 819         int i;
 820
 821         if (strcmp(key, "class") == 0)
 822                 return 0;
 823         if (strcmp(key, "wcs_file") == 0) {
 824                 file = fopen(val, "rb");
 825                 if (file == NULL) {
 826                         rte_errno = ENOTSUP;
 827                         return -rte_errno;
 828                 }
 829                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 830                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 831                         if (ret <= 0) {
 832                                 fclose(file);
 833                                 DRV_LOG(ERR,
 834                                         "Failed to read credential from file.");
 835                                 rte_errno = EINVAL;
 836                                 return -rte_errno;
 837                         }
 838                 }
 839                 fclose(file);
 840                 devarg_prms->login_devarg = true;
 841                 return 0;
 842         }
 843         errno = 0;
 844         tmp = strtoul(val, NULL, 0);
 845         if (errno) {
 846                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 847                 return -errno;
 848         }
 849         if (strcmp(key, "max_segs_num") == 0) {
 850                 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
 851                         DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
 852                                 " be less than %d.",
 853                                 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
 854                         rte_errno = EINVAL;
 855                         return -rte_errno;
 856                 }
 857                 devarg_prms->max_segs_num = (uint32_t)tmp;
 858         } else if (strcmp(key, "import_kek_id") == 0) {
 859                 attr->session_import_kek_ptr = (uint32_t)tmp;
 860         } else if (strcmp(key, "credential_id") == 0) {
 861                 attr->credential_pointer = (uint32_t)tmp;
 862         } else if (strcmp(key, "keytag") == 0) {
 863                 devarg_prms->keytag = tmp;
 864         } else {
 865                 DRV_LOG(WARNING, "Invalid key %s.", key);
 866         }
 867         return 0;
 868 }
 869
 870 static int
 871 mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 872                           struct mlx5_crypto_devarg_params *devarg_prms)
 873 {
 874         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 875         struct rte_kvargs *kvlist;
 876
 877         /* Default values. */
 878         attr->credential_pointer = 0;
 879         attr->session_import_kek_ptr = 0;
 880         devarg_prms->keytag = 0;
 881         devarg_prms->max_segs_num = 8;
 882         if (devargs == NULL) {
 883                 DRV_LOG(ERR,
 884         "No login devargs in order to enable crypto operations in the device.");
 885                 rte_errno = EINVAL;
 886                 return -1;
 887         }
 888         kvlist = rte_kvargs_parse(devargs->args, NULL);
 889         if (kvlist == NULL) {
 890                 DRV_LOG(ERR, "Failed to parse devargs.");
 891                 rte_errno = EINVAL;
 892                 return -1;
 893         }
 894         if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
 895                            devarg_prms) != 0) {
 896                 DRV_LOG(ERR, "Devargs handler function Failed.");
 897                 rte_kvargs_free(kvlist);
 898                 rte_errno = EINVAL;
 899                 return -1;
 900         }
 901         rte_kvargs_free(kvlist);
 902         if (devarg_prms->login_devarg == false) {
 903                 DRV_LOG(ERR,
 904         "No login credential devarg in order to enable crypto operations "
 905         "in the device.");
 906                 rte_errno = EINVAL;
 907                 return -1;
 908         }
 909         return 0;
 910 }
 911
 912 /**
 913  * Callback for memory event.
 914  *
 915  * @param event_type
 916  *   Memory event type.
 917  * @param addr
 918  *   Address of memory.
 919  * @param len
 920  *   Size of memory.
 921  */
 922 static void
 923 mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 924                             size_t len, void *arg __rte_unused)
 925 {
 926         struct mlx5_crypto_priv *priv;
 927
 928         /* Must be called from the primary process. */
 929         MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
 930         switch (event_type) {
 931         case RTE_MEM_EVENT_FREE:
 932                 pthread_mutex_lock(&priv_list_lock);
 933                 /* Iterate all the existing mlx5 devices. */
 934                 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 935                         mlx5_free_mr_by_addr(&priv->mr_scache,
 936                                              priv->ctx->device->name,
 937                                              addr, len);
 938                 pthread_mutex_unlock(&priv_list_lock);
 939                 break;
 940         case RTE_MEM_EVENT_ALLOC:
 941         default:
 942                 break;
 943         }
 944 }
 945
 946 /**
 947  * DPDK callback to register a PCI device.
 948  *
 949  * This function spawns crypto device out of a given PCI device.
 950  *
 951  * @param[in] pci_drv
 952  *   PCI driver structure (mlx5_crypto_driver).
 953  * @param[in] pci_dev
 954  *   PCI device information.
 955  *
 956  * @return
 957  *   0 on success, 1 to skip this driver, a negative errno value otherwise
 958  *   and rte_errno is set.
 959  */
 960 static int
 961 mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 962                         struct rte_pci_device *pci_dev)
 963 {
 964         struct ibv_device *ibv;
 965         struct rte_cryptodev *crypto_dev;
 966         struct ibv_context *ctx;
 967         struct mlx5_devx_obj *login;
 968         struct mlx5_crypto_priv *priv;
 969         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 970         struct mlx5_hca_attr attr = { 0 };
 971         struct rte_cryptodev_pmd_init_params init_params = {
 972                 .name = "",
 973                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 974                 .socket_id = pci_dev->device.numa_node,
 975                 .max_nb_queue_pairs =
 976                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 977         };
 978         uint16_t rdmw_wqe_size;
 979         int ret;
 980
 981         RTE_SET_USED(pci_drv);
 982         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 983                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 984                 rte_errno = ENOTSUP;
 985                 return -rte_errno;
 986         }
 987         ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
 988         if (ibv == NULL) {
 989                 DRV_LOG(ERR, "No matching IB device for PCI slot "
 990                         PCI_PRI_FMT ".", pci_dev->addr.domain,
 991                         pci_dev->addr.bus, pci_dev->addr.devid,
 992                         pci_dev->addr.function);
 993                 return -rte_errno;
 994         }
 995         DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
 996         ctx = mlx5_glue->dv_open_device(ibv);
 997         if (ctx == NULL) {
 998                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 999                 rte_errno = ENODEV;
1000                 return -rte_errno;
1001         }
1002         if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
1003             attr.crypto == 0 || attr.aes_xts == 0) {
1004                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
1005                         "operations, maybe old FW/OFED version?");
1006                 claim_zero(mlx5_glue->close_device(ctx));
1007                 rte_errno = ENOTSUP;
1008                 return -ENOTSUP;
1009         }
1010         ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
1011         if (ret) {
1012                 DRV_LOG(ERR, "Failed to parse devargs.");
1013                 return -rte_errno;
1014         }
1015         login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
1016                                                       &devarg_prms.login_attr);
1017         if (login == NULL) {
1018                 DRV_LOG(ERR, "Failed to configure login.");
1019                 return -rte_errno;
1020         }
1021         crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
1022                                         &init_params);
1023         if (crypto_dev == NULL) {
1024                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
1025                 claim_zero(mlx5_glue->close_device(ctx));
1026                 return -ENODEV;
1027         }
1028         DRV_LOG(INFO,
1029                 "Crypto device %s was created successfully.", ibv->name);
1030         crypto_dev->dev_ops = &mlx5_crypto_ops;
1031         crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
1032         crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
1033         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
1034         crypto_dev->driver_id = mlx5_crypto_driver_id;
1035         priv = crypto_dev->data->dev_private;
1036         priv->ctx = ctx;
1037         priv->login_obj = login;
1038         priv->pci_dev = pci_dev;
1039         priv->crypto_dev = crypto_dev;
1040         if (mlx5_crypto_hw_global_prepare(priv) != 0) {
1041                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
1042                 claim_zero(mlx5_glue->close_device(priv->ctx));
1043                 return -1;
1044         }
1045         if (mlx5_mr_btree_init(&priv->mr_scache.cache,
1046                              MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
1047                 DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
1048                 mlx5_crypto_hw_global_release(priv);
1049                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
1050                 claim_zero(mlx5_glue->close_device(priv->ctx));
1051                 rte_errno = ENOMEM;
1052                 return -rte_errno;
1053         }
1054         priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
1055         priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
1056         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
1057         priv->max_segs_num = devarg_prms.max_segs_num;
1058         priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
1059                              sizeof(struct mlx5_umr_wqe) +
1060                              RTE_ALIGN(priv->max_segs_num, 4) *
1061                              sizeof(struct mlx5_wqe_dseg);
1062         rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
1063                               sizeof(struct mlx5_wqe_dseg) *
1064                               (priv->max_segs_num <= 2 ? 2 : 2 +
1065                                RTE_ALIGN(priv->max_segs_num - 2, 4));
1066         priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
1067         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
1068         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
1069         /* Register callback function for global shared MR cache management. */
1070         if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
1071                 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
1072                                                 mlx5_crypto_mr_mem_event_cb,
1073                                                 NULL);
1074         pthread_mutex_lock(&priv_list_lock);
1075         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
1076         pthread_mutex_unlock(&priv_list_lock);
1077         return 0;
1078 }
1079
1080 static int
1081 mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
1082 {
1083         struct mlx5_crypto_priv *priv = NULL;
1084
1085         pthread_mutex_lock(&priv_list_lock);
1086         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
1087                 if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
1088                         break;
1089         if (priv)
1090                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
1091         pthread_mutex_unlock(&priv_list_lock);
1092         if (priv) {
1093                 if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
1094                         rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
1095                                                           NULL);
1096                 mlx5_mr_release_cache(&priv->mr_scache);
1097                 mlx5_crypto_hw_global_release(priv);
1098                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
1099                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
1100                 claim_zero(mlx5_glue->close_device(priv->ctx));
1101         }
1102         return 0;
1103 }
1104
1105 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
1106                 {
1107                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
1108                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
1109                 },
1110                 {
1111                         .vendor_id = 0
1112                 }
1113 };
1114
1115 static struct mlx5_pci_driver mlx5_crypto_driver = {
1116         .driver_class = MLX5_CLASS_CRYPTO,
1117         .pci_driver = {
1118                 .driver = {
1119                         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
1120                 },
1121                 .id_table = mlx5_crypto_pci_id_map,
1122                 .probe = mlx5_crypto_pci_probe,
1123                 .remove = mlx5_crypto_pci_remove,
1124                 .drv_flags = 0,
1125         },
1126 };
1127
1128 RTE_INIT(rte_mlx5_crypto_init)
1129 {
1130         mlx5_common_init();
1131         if (mlx5_glue != NULL)
1132                 mlx5_pci_driver_register(&mlx5_crypto_driver);
1133 }
1134
1135 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
1136                                mlx5_crypto_driver_id);
1137
1138 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
1139 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
1140 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
1141 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");