drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_eal_paging.h>
   8 #include <rte_errno.h>
   9 #include <rte_log.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_memory.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 1024
  24 #define MLX5_CRYPTO_MAX_SEGS 56
  25
  26 #define MLX5_CRYPTO_FEATURE_FLAGS \
  27         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  28          RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
  29          RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
  30          RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
  31          RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
  32          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  33          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  34
  35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  36                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  37 static pthread_mutex_t priv_list_lock;
  38
  39 int mlx5_crypto_logtype;
  40
  41 uint8_t mlx5_crypto_driver_id;
  42
  43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  44         {               /* AES XTS */
  45                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  46                 {.sym = {
  47                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  48                         {.cipher = {
  49                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  50                                 .block_size = 16,
  51                                 .key_size = {
  52                                         .min = 32,
  53                                         .max = 64,
  54                                         .increment = 32
  55                                 },
  56                                 .iv_size = {
  57                                         .min = 16,
  58                                         .max = 16,
  59                                         .increment = 0
  60                                 },
  61                                 .dataunit_set =
  62                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  63                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES |
  64                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES,
  65                         }, }
  66                 }, }
  67         },
  68 };
  69
  70 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  71
  72 static const struct rte_driver mlx5_drv = {
  73         .name = mlx5_crypto_drv_name,
  74         .alias = mlx5_crypto_drv_name
  75 };
  76
  77 static struct cryptodev_driver mlx5_cryptodev_driver;
  78
  79 struct mlx5_crypto_session {
  80         uint32_t bs_bpt_eo_es;
  81         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  82          * saved in big endian format.
  83          */
  84         uint32_t bsp_res;
  85         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  86          * endian format.
  87          */
  88         uint32_t iv_offset:16;
  89         /**< Starting point for Initialisation Vector. */
  90         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  91         uint32_t dek_id; /**< DEK ID */
  92 } __rte_packed;
  93
  94 static void
  95 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  96                           struct rte_cryptodev_info *dev_info)
  97 {
  98         RTE_SET_USED(dev);
  99         if (dev_info != NULL) {
 100                 dev_info->driver_id = mlx5_crypto_driver_id;
 101                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 102                 dev_info->capabilities = mlx5_crypto_caps;
 103                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 104                 dev_info->min_mbuf_headroom_req = 0;
 105                 dev_info->min_mbuf_tailroom_req = 0;
 106                 dev_info->sym.max_nb_sessions = 0;
 107                 /*
 108                  * If 0, the device does not have any limitation in number of
 109                  * sessions that can be used.
 110                  */
 111         }
 112 }
 113
 114 static int
 115 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 116                           struct rte_cryptodev_config *config)
 117 {
 118         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 119
 120         if (config == NULL) {
 121                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 122                 return -EINVAL;
 123         }
 124         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 125                 DRV_LOG(ERR,
 126                         "Disabled symmetric crypto feature is not supported.");
 127                 return -ENOTSUP;
 128         }
 129         if (mlx5_crypto_dek_setup(priv) != 0) {
 130                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 131                 return -ENOMEM;
 132         }
 133         priv->dev_config = *config;
 134         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 135         return 0;
 136 }
 137
 138 static void
 139 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 140 {
 141         RTE_SET_USED(dev);
 142 }
 143
 144 static int
 145 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 146 {
 147         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 148
 149         return mlx5_dev_mempool_subscribe(priv->cdev);
 150 }
 151
 152 static int
 153 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 154 {
 155         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 156
 157         mlx5_crypto_dek_unset(priv);
 158         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 159         return 0;
 160 }
 161
 162 static unsigned int
 163 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 164 {
 165         return sizeof(struct mlx5_crypto_session);
 166 }
 167
 168 static int
 169 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 170                                   struct rte_crypto_sym_xform *xform,
 171                                   struct rte_cryptodev_sym_session *session,
 172                                   struct rte_mempool *mp)
 173 {
 174         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 175         struct mlx5_crypto_session *sess_private_data;
 176         struct rte_crypto_cipher_xform *cipher;
 177         uint8_t encryption_order;
 178         int ret;
 179
 180         if (unlikely(xform->next != NULL)) {
 181                 DRV_LOG(ERR, "Xform next is not supported.");
 182                 return -ENOTSUP;
 183         }
 184         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 185                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 186                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 187                 return -ENOTSUP;
 188         }
 189         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 190         if (ret != 0) {
 191                 DRV_LOG(ERR,
 192                         "Failed to get session %p private data from mempool.",
 193                         sess_private_data);
 194                 return -ENOMEM;
 195         }
 196         cipher = &xform->cipher;
 197         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 198         if (sess_private_data->dek == NULL) {
 199                 rte_mempool_put(mp, sess_private_data);
 200                 DRV_LOG(ERR, "Failed to prepare dek.");
 201                 return -ENOMEM;
 202         }
 203         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 204                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 205         else
 206                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 207         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 208                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 209                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 210                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 211                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 212         switch (xform->cipher.dataunit_len) {
 213         case 0:
 214                 sess_private_data->bsp_res = 0;
 215                 break;
 216         case 512:
 217                 sess_private_data->bsp_res = rte_cpu_to_be_32
 218                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 219                                              MLX5_BLOCK_SIZE_OFFSET);
 220                 break;
 221         case 4096:
 222                 sess_private_data->bsp_res = rte_cpu_to_be_32
 223                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 224                                              MLX5_BLOCK_SIZE_OFFSET);
 225                 break;
 226         case 1048576:
 227                 sess_private_data->bsp_res = rte_cpu_to_be_32
 228                                              ((uint32_t)MLX5_BLOCK_SIZE_1MB <<
 229                                              MLX5_BLOCK_SIZE_OFFSET);
 230                 break;
 231         default:
 232                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 233                 return -ENOTSUP;
 234         }
 235         sess_private_data->iv_offset = cipher->iv.offset;
 236         sess_private_data->dek_id =
 237                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 238                                          0xffffff);
 239         set_sym_session_private_data(session, dev->driver_id,
 240                                      sess_private_data);
 241         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 242         return 0;
 243 }
 244
 245 static void
 246 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 247                               struct rte_cryptodev_sym_session *sess)
 248 {
 249         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 250         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 251                                                                 dev->driver_id);
 252
 253         if (unlikely(spriv == NULL)) {
 254                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 255                 return;
 256         }
 257         mlx5_crypto_dek_destroy(priv, spriv->dek);
 258         set_sym_session_private_data(sess, dev->driver_id, NULL);
 259         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 260         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 261 }
 262
 263 static void
 264 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n)
 265 {
 266         uint16_t i;
 267
 268         for (i = 0; i < n; i++)
 269                 if (qp->mkey[i])
 270                         claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
 271 }
 272
 273 static void
 274 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp)
 275 {
 276         if (qp == NULL)
 277                 return;
 278         mlx5_devx_qp_destroy(&qp->qp_obj);
 279         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 280         mlx5_devx_cq_destroy(&qp->cq_obj);
 281         rte_free(qp);
 282 }
 283
 284 static int
 285 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 286 {
 287         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 288
 289         mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
 290         mlx5_crypto_qp_release(qp);
 291         dev->data->queue_pairs[qp_id] = NULL;
 292         return 0;
 293 }
 294
 295 static __rte_noinline uint32_t
 296 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 297 {
 298         uint32_t bl = op->sym->cipher.data.length;
 299
 300         switch (bl) {
 301         case (1 << 20):
 302                 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
 303         case (1 << 12):
 304                 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
 305                                 MLX5_BLOCK_SIZE_OFFSET);
 306         case (1 << 9):
 307                 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
 308         default:
 309                 DRV_LOG(ERR, "Unknown block size: %u.", bl);
 310                 return UINT32_MAX;
 311         }
 312 }
 313
 314 static __rte_always_inline uint32_t
 315 mlx5_crypto_klm_set(struct mlx5_crypto_qp *qp, struct rte_mbuf *mbuf,
 316                     struct mlx5_wqe_dseg *klm, uint32_t offset,
 317                     uint32_t *remain)
 318 {
 319         uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 320         uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 321
 322         if (data_len > *remain)
 323                 data_len = *remain;
 324         *remain -= data_len;
 325         klm->bcount = rte_cpu_to_be_32(data_len);
 326         klm->pbuf = rte_cpu_to_be_64(addr);
 327         klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
 328         return klm->lkey;
 329
 330 }
 331
 332 static __rte_always_inline uint32_t
 333 mlx5_crypto_klms_set(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op,
 334                      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm)
 335 {
 336         uint32_t remain_len = op->sym->cipher.data.length;
 337         uint32_t nb_segs = mbuf->nb_segs;
 338         uint32_t klm_n = 1u;
 339
 340         /* First mbuf needs to take the cipher offset. */
 341         if (unlikely(mlx5_crypto_klm_set(qp, mbuf, klm,
 342                      op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 343                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 344                 return 0;
 345         }
 346         while (remain_len) {
 347                 nb_segs--;
 348                 mbuf = mbuf->next;
 349                 if (unlikely(mbuf == NULL || nb_segs == 0)) {
 350                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 351                         return 0;
 352                 }
 353                 if (unlikely(mlx5_crypto_klm_set(qp, mbuf, ++klm, 0,
 354                                                  &remain_len) == UINT32_MAX)) {
 355                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 356                         return 0;
 357                 }
 358                 klm_n++;
 359         }
 360         return klm_n;
 361 }
 362
 363 static __rte_always_inline int
 364 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 365                          struct mlx5_crypto_qp *qp,
 366                          struct rte_crypto_op *op,
 367                          struct mlx5_umr_wqe *umr)
 368 {
 369         struct mlx5_crypto_session *sess = get_sym_session_private_data
 370                                 (op->sym->session, mlx5_crypto_driver_id);
 371         struct mlx5_wqe_cseg *cseg = &umr->ctr;
 372         struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
 373         struct mlx5_wqe_dseg *klms = &umr->kseg[0];
 374         struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
 375                                       RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
 376         uint32_t ds;
 377         bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 378         /* Set UMR WQE. */
 379         uint32_t klm_n = mlx5_crypto_klms_set(qp, op,
 380                                    ipl ? op->sym->m_src : op->sym->m_dst, klms);
 381
 382         if (unlikely(klm_n == 0))
 383                 return 0;
 384         bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
 385         if (unlikely(!sess->bsp_res)) {
 386                 bsf->bsp_res = mlx5_crypto_get_block_size(op);
 387                 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
 388                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 389                         return 0;
 390                 }
 391         } else {
 392                 bsf->bsp_res = sess->bsp_res;
 393         }
 394         bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
 395         memcpy(bsf->xts_initial_tweak,
 396                rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
 397         bsf->res_dp = sess->dek_id;
 398         mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
 399         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
 400         qp->db_pi += priv->umr_wqe_stride;
 401         /* Set RDMA_WRITE WQE. */
 402         cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 403         klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 404         if (!ipl) {
 405                 klm_n = mlx5_crypto_klms_set(qp, op, op->sym->m_src, klms);
 406                 if (unlikely(klm_n == 0))
 407                         return 0;
 408         } else {
 409                 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
 410         }
 411         ds = 2 + klm_n;
 412         cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 413         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 414                                                         MLX5_OPCODE_RDMA_WRITE);
 415         ds = RTE_ALIGN(ds, 4);
 416         qp->db_pi += ds >> 2;
 417         /* Set NOP WQE if needed. */
 418         if (priv->max_rdmar_ds > ds) {
 419                 cseg += ds;
 420                 ds = priv->max_rdmar_ds - ds;
 421                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 422                 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 423                                                                MLX5_OPCODE_NOP);
 424                 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
 425         }
 426         qp->wqe = (uint8_t *)cseg;
 427         return 1;
 428 }
 429
 430 static uint16_t
 431 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 432                           uint16_t nb_ops)
 433 {
 434         struct mlx5_crypto_qp *qp = queue_pair;
 435         struct mlx5_crypto_priv *priv = qp->priv;
 436         struct mlx5_umr_wqe *umr;
 437         struct rte_crypto_op *op;
 438         uint16_t mask = qp->entries_n - 1;
 439         uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
 440         uint32_t idx;
 441
 442         if (remain < nb_ops)
 443                 nb_ops = remain;
 444         else
 445                 remain = nb_ops;
 446         if (unlikely(remain == 0))
 447                 return 0;
 448         do {
 449                 idx = qp->pi & mask;
 450                 op = *ops++;
 451                 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 452                         priv->wqe_set_size * idx);
 453                 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
 454                         qp->stats.enqueue_err_count++;
 455                         if (remain != nb_ops) {
 456                                 qp->stats.enqueued_count -= remain;
 457                                 break;
 458                         }
 459                         return 0;
 460                 }
 461                 qp->ops[idx] = op;
 462                 qp->pi++;
 463         } while (--remain);
 464         qp->stats.enqueued_count += nb_ops;
 465         mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)qp->wqe,
 466                            qp->db_pi, &qp->qp_obj.db_rec[MLX5_SND_DBR],
 467                            !priv->uar.dbnc);
 468         return nb_ops;
 469 }
 470
 471 static __rte_noinline void
 472 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 473 {
 474         const uint32_t idx = qp->ci & (qp->entries_n - 1);
 475         volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
 476                                                         &qp->cq_obj.cqes[idx];
 477
 478         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 479         qp->stats.dequeue_err_count++;
 480         DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 481 }
 482
 483 static uint16_t
 484 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 485                           uint16_t nb_ops)
 486 {
 487         struct mlx5_crypto_qp *qp = queue_pair;
 488         volatile struct mlx5_cqe *restrict cqe;
 489         struct rte_crypto_op *restrict op;
 490         const unsigned int cq_size = qp->entries_n;
 491         const unsigned int mask = cq_size - 1;
 492         uint32_t idx;
 493         uint32_t next_idx = qp->ci & mask;
 494         const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
 495         uint16_t i = 0;
 496         int ret;
 497
 498         if (unlikely(max == 0))
 499                 return 0;
 500         do {
 501                 idx = next_idx;
 502                 next_idx = (qp->ci + 1) & mask;
 503                 op = qp->ops[idx];
 504                 cqe = &qp->cq_obj.cqes[idx];
 505                 ret = check_cqe(cqe, cq_size, qp->ci);
 506                 rte_io_rmb();
 507                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 508                         if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
 509                                 mlx5_crypto_cqe_err_handle(qp, op);
 510                         break;
 511                 }
 512                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 513                 ops[i++] = op;
 514                 qp->ci++;
 515         } while (i < max);
 516         if (likely(i != 0)) {
 517                 rte_io_wmb();
 518                 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
 519                 qp->stats.dequeued_count += i;
 520         }
 521         return i;
 522 }
 523
 524 static void
 525 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 526 {
 527         uint32_t i;
 528
 529         for (i = 0 ; i < qp->entries_n; i++) {
 530                 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 531                         i * priv->wqe_set_size);
 532                 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
 533                                                                      (cseg + 1);
 534                 struct mlx5_wqe_umr_bsf_seg *bsf =
 535                         (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
 536                                                        priv->umr_wqe_size)) - 1;
 537                 struct mlx5_wqe_rseg *rseg;
 538
 539                 /* Init UMR WQE. */
 540                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
 541                                          (priv->umr_wqe_size / MLX5_WSEG_SIZE));
 542                 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 543                                        MLX5_COMP_MODE_OFFSET);
 544                 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
 545                 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
 546                 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
 547                 ucseg->ko_to_bs = rte_cpu_to_be_32
 548                         ((MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size) <<
 549                          MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
 550                 bsf->keytag = priv->keytag;
 551                 /* Init RDMA WRITE WQE. */
 552                 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 553                 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
 554                                       MLX5_COMP_MODE_OFFSET) |
 555                                       MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
 556                 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
 557                 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
 558         }
 559 }
 560
 561 static int
 562 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
 563                                   struct mlx5_crypto_qp *qp)
 564 {
 565         struct mlx5_umr_wqe *umr;
 566         uint32_t i;
 567         struct mlx5_devx_mkey_attr attr = {
 568                 .pd = priv->cdev->pdn,
 569                 .umr_en = 1,
 570                 .crypto_en = 1,
 571                 .set_remote_rw = 1,
 572                 .klm_num = MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
 573         };
 574
 575         for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
 576            i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
 577                 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
 578                 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
 579                 if (!qp->mkey[i])
 580                         goto error;
 581         }
 582         return 0;
 583 error:
 584         DRV_LOG(ERR, "Failed to allocate indirect mkey.");
 585         mlx5_crypto_indirect_mkeys_release(qp, i);
 586         return -1;
 587 }
 588
 589 static int
 590 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 591                              const struct rte_cryptodev_qp_conf *qp_conf,
 592                              int socket_id)
 593 {
 594         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 595         struct mlx5_devx_qp_attr attr = {0};
 596         struct mlx5_crypto_qp *qp;
 597         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 598         uint32_t ret;
 599         uint32_t alloc_size = sizeof(*qp);
 600         uint32_t log_wqbb_n;
 601         struct mlx5_devx_cq_attr cq_attr = {
 602                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar.obj),
 603         };
 604
 605         if (dev->data->queue_pairs[qp_id] != NULL)
 606                 mlx5_crypto_queue_pair_release(dev, qp_id);
 607         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 608         alloc_size += (sizeof(struct rte_crypto_op *) +
 609                        sizeof(struct mlx5_devx_obj *)) *
 610                        RTE_BIT32(log_nb_desc);
 611         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 612                                 socket_id);
 613         if (qp == NULL) {
 614                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 615                 rte_errno = ENOMEM;
 616                 return -rte_errno;
 617         }
 618         if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc,
 619                                 &cq_attr, socket_id) != 0) {
 620                 DRV_LOG(ERR, "Failed to create CQ.");
 621                 goto error;
 622         }
 623         log_wqbb_n = rte_log2_u32(RTE_BIT32(log_nb_desc) *
 624                                 (priv->wqe_set_size / MLX5_SEND_WQE_BB));
 625         attr.pd = priv->cdev->pdn;
 626         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar.obj);
 627         attr.cqn = qp->cq_obj.cq->id;
 628         attr.num_of_receive_wqes = 0;
 629         attr.num_of_send_wqbbs = RTE_BIT32(log_wqbb_n);
 630         attr.ts_format =
 631                 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 632         ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj,
 633                                         attr.num_of_send_wqbbs * MLX5_WQE_SIZE,
 634                                         &attr, socket_id);
 635         if (ret) {
 636                 DRV_LOG(ERR, "Failed to create QP.");
 637                 goto error;
 638         }
 639         if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
 640                               priv->dev_config.socket_id) != 0) {
 641                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 642                         (uint32_t)qp_id);
 643                 rte_errno = ENOMEM;
 644                 goto error;
 645         }
 646         /*
 647          * In Order to configure self loopback, when calling devx qp2rts the
 648          * remote QP id that is used is the id of the same QP.
 649          */
 650         if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id))
 651                 goto error;
 652         qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 653                                                            RTE_CACHE_LINE_SIZE);
 654         qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
 655         qp->entries_n = 1 << log_nb_desc;
 656         if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
 657                 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
 658                 rte_errno = ENOMEM;
 659                 goto error;
 660         }
 661         mlx5_crypto_qp_init(priv, qp);
 662         qp->priv = priv;
 663         dev->data->queue_pairs[qp_id] = qp;
 664         return 0;
 665 error:
 666         mlx5_crypto_qp_release(qp);
 667         return -1;
 668 }
 669
 670 static void
 671 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
 672                       struct rte_cryptodev_stats *stats)
 673 {
 674         int qp_id;
 675
 676         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 677                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 678
 679                 stats->enqueued_count += qp->stats.enqueued_count;
 680                 stats->dequeued_count += qp->stats.dequeued_count;
 681                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 682                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 683         }
 684 }
 685
 686 static void
 687 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
 688 {
 689         int qp_id;
 690
 691         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 692                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 693
 694                 memset(&qp->stats, 0, sizeof(qp->stats));
 695         }
 696 }
 697
 698 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 699         .dev_configure                  = mlx5_crypto_dev_configure,
 700         .dev_start                      = mlx5_crypto_dev_start,
 701         .dev_stop                       = mlx5_crypto_dev_stop,
 702         .dev_close                      = mlx5_crypto_dev_close,
 703         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 704         .stats_get                      = mlx5_crypto_stats_get,
 705         .stats_reset                    = mlx5_crypto_stats_reset,
 706         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 707         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 708         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 709         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 710         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 711         .sym_get_raw_dp_ctx_size        = NULL,
 712         .sym_configure_raw_dp_ctx       = NULL,
 713 };
 714
 715 static int
 716 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 717 {
 718         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 719         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 720         unsigned long tmp;
 721         FILE *file;
 722         int ret;
 723         int i;
 724
 725         if (strcmp(key, "class") == 0)
 726                 return 0;
 727         if (strcmp(key, "wcs_file") == 0) {
 728                 file = fopen(val, "rb");
 729                 if (file == NULL) {
 730                         rte_errno = ENOTSUP;
 731                         return -rte_errno;
 732                 }
 733                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 734                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 735                         if (ret <= 0) {
 736                                 fclose(file);
 737                                 DRV_LOG(ERR,
 738                                         "Failed to read credential from file.");
 739                                 rte_errno = EINVAL;
 740                                 return -rte_errno;
 741                         }
 742                 }
 743                 fclose(file);
 744                 devarg_prms->login_devarg = true;
 745                 return 0;
 746         }
 747         errno = 0;
 748         tmp = strtoul(val, NULL, 0);
 749         if (errno) {
 750                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 751                 return -errno;
 752         }
 753         if (strcmp(key, "max_segs_num") == 0) {
 754                 if (!tmp) {
 755                         DRV_LOG(ERR, "max_segs_num must be greater than 0.");
 756                         rte_errno = EINVAL;
 757                         return -rte_errno;
 758                 }
 759                 devarg_prms->max_segs_num = (uint32_t)tmp;
 760         } else if (strcmp(key, "import_kek_id") == 0) {
 761                 attr->session_import_kek_ptr = (uint32_t)tmp;
 762         } else if (strcmp(key, "credential_id") == 0) {
 763                 attr->credential_pointer = (uint32_t)tmp;
 764         } else if (strcmp(key, "keytag") == 0) {
 765                 devarg_prms->keytag = tmp;
 766         } else {
 767                 DRV_LOG(WARNING, "Invalid key %s.", key);
 768         }
 769         return 0;
 770 }
 771
 772 static int
 773 mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 774                           struct mlx5_crypto_devarg_params *devarg_prms)
 775 {
 776         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 777         struct rte_kvargs *kvlist;
 778
 779         /* Default values. */
 780         attr->credential_pointer = 0;
 781         attr->session_import_kek_ptr = 0;
 782         devarg_prms->keytag = 0;
 783         devarg_prms->max_segs_num = 8;
 784         if (devargs == NULL) {
 785                 DRV_LOG(ERR,
 786         "No login devargs in order to enable crypto operations in the device.");
 787                 rte_errno = EINVAL;
 788                 return -1;
 789         }
 790         kvlist = rte_kvargs_parse(devargs->args, NULL);
 791         if (kvlist == NULL) {
 792                 DRV_LOG(ERR, "Failed to parse devargs.");
 793                 rte_errno = EINVAL;
 794                 return -1;
 795         }
 796         if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
 797                            devarg_prms) != 0) {
 798                 DRV_LOG(ERR, "Devargs handler function Failed.");
 799                 rte_kvargs_free(kvlist);
 800                 rte_errno = EINVAL;
 801                 return -1;
 802         }
 803         rte_kvargs_free(kvlist);
 804         if (devarg_prms->login_devarg == false) {
 805                 DRV_LOG(ERR,
 806         "No login credential devarg in order to enable crypto operations "
 807         "in the device.");
 808                 rte_errno = EINVAL;
 809                 return -1;
 810         }
 811         return 0;
 812 }
 813
 814 /*
 815  * Calculate UMR WQE size and RDMA Write WQE size with the
 816  * following limitations:
 817  *      - Each WQE size is multiple of 64.
 818  *      - The summarize of both UMR WQE and RDMA_W WQE is a power of 2.
 819  *      - The number of entries in the UMR WQE's KLM list is multiple of 4.
 820  */
 821 static void
 822 mlx5_crypto_get_wqe_sizes(uint32_t segs_num, uint32_t *umr_size,
 823                         uint32_t *rdmaw_size)
 824 {
 825         uint32_t diff, wqe_set_size;
 826
 827         *umr_size = MLX5_CRYPTO_UMR_WQE_STATIC_SIZE +
 828                         RTE_ALIGN(segs_num, 4) *
 829                         sizeof(struct mlx5_wqe_dseg);
 830         /* Make sure UMR WQE size is multiple of WQBB. */
 831         *umr_size = RTE_ALIGN(*umr_size, MLX5_SEND_WQE_BB);
 832         *rdmaw_size = sizeof(struct mlx5_rdma_write_wqe) +
 833                         sizeof(struct mlx5_wqe_dseg) *
 834                         (segs_num <= 2 ? 2 : 2 +
 835                         RTE_ALIGN(segs_num - 2, 4));
 836         /* Make sure RDMA_WRITE WQE size is multiple of WQBB. */
 837         *rdmaw_size = RTE_ALIGN(*rdmaw_size, MLX5_SEND_WQE_BB);
 838         wqe_set_size = *rdmaw_size + *umr_size;
 839         diff = rte_align32pow2(wqe_set_size) - wqe_set_size;
 840         /* Make sure wqe_set size is power of 2. */
 841         if (diff)
 842                 *umr_size += diff;
 843 }
 844
 845 static uint8_t
 846 mlx5_crypto_max_segs_num(uint16_t max_wqe_size)
 847 {
 848         int klms_sizes = max_wqe_size - MLX5_CRYPTO_UMR_WQE_STATIC_SIZE;
 849         uint32_t max_segs_cap = RTE_ALIGN_FLOOR(klms_sizes, MLX5_SEND_WQE_BB) /
 850                         sizeof(struct mlx5_wqe_dseg);
 851
 852         MLX5_ASSERT(klms_sizes >= MLX5_SEND_WQE_BB);
 853         while (max_segs_cap) {
 854                 uint32_t umr_wqe_size, rdmw_wqe_size;
 855
 856                 mlx5_crypto_get_wqe_sizes(max_segs_cap, &umr_wqe_size,
 857                                                 &rdmw_wqe_size);
 858                 if (umr_wqe_size <= max_wqe_size &&
 859                                 rdmw_wqe_size <= max_wqe_size)
 860                         break;
 861                 max_segs_cap -= 4;
 862         }
 863         return max_segs_cap;
 864 }
 865
 866 static int
 867 mlx5_crypto_configure_wqe_size(struct mlx5_crypto_priv *priv,
 868                                 uint16_t max_wqe_size, uint32_t max_segs_num)
 869 {
 870         uint32_t rdmw_wqe_size, umr_wqe_size;
 871
 872         mlx5_crypto_get_wqe_sizes(max_segs_num, &umr_wqe_size,
 873                                         &rdmw_wqe_size);
 874         priv->wqe_set_size = rdmw_wqe_size + umr_wqe_size;
 875         if (umr_wqe_size > max_wqe_size ||
 876                                 rdmw_wqe_size > max_wqe_size) {
 877                 DRV_LOG(ERR, "Invalid max_segs_num: %u. should be %u or lower.",
 878                         max_segs_num,
 879                         mlx5_crypto_max_segs_num(max_wqe_size));
 880                 rte_errno = EINVAL;
 881                 return -EINVAL;
 882         }
 883         priv->umr_wqe_size = (uint16_t)umr_wqe_size;
 884         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
 885         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 886         return 0;
 887 }
 888
 889 static int
 890 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 891 {
 892         struct rte_cryptodev *crypto_dev;
 893         struct mlx5_devx_obj *login;
 894         struct mlx5_crypto_priv *priv;
 895         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 896         struct rte_cryptodev_pmd_init_params init_params = {
 897                 .name = "",
 898                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 899                 .socket_id = cdev->dev->numa_node,
 900                 .max_nb_queue_pairs =
 901                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 902         };
 903         const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
 904         int ret;
 905
 906         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 907                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 908                 rte_errno = ENOTSUP;
 909                 return -rte_errno;
 910         }
 911         if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
 912                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 913                         "operations, maybe old FW/OFED version?");
 914                 rte_errno = ENOTSUP;
 915                 return -ENOTSUP;
 916         }
 917         ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms);
 918         if (ret) {
 919                 DRV_LOG(ERR, "Failed to parse devargs.");
 920                 return -rte_errno;
 921         }
 922         crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
 923                                               &init_params);
 924         if (crypto_dev == NULL) {
 925                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
 926                 return -ENODEV;
 927         }
 928         DRV_LOG(INFO,
 929                 "Crypto device %s was created successfully.", ibdev_name);
 930         crypto_dev->dev_ops = &mlx5_crypto_ops;
 931         crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 932         crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 933         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 934         crypto_dev->driver_id = mlx5_crypto_driver_id;
 935         priv = crypto_dev->data->dev_private;
 936         priv->cdev = cdev;
 937         priv->crypto_dev = crypto_dev;
 938         if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
 939                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 940                 return -1;
 941         }
 942         login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
 943                                                       &devarg_prms.login_attr);
 944         if (login == NULL) {
 945                 DRV_LOG(ERR, "Failed to configure login.");
 946                 mlx5_devx_uar_release(&priv->uar);
 947                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 948                 return -rte_errno;
 949         }
 950         priv->login_obj = login;
 951         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 952         ret = mlx5_crypto_configure_wqe_size(priv,
 953                 cdev->config.hca_attr.max_wqe_sz_sq, devarg_prms.max_segs_num);
 954         if (ret) {
 955                 mlx5_devx_uar_release(&priv->uar);
 956                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 957                 return -1;
 958         }
 959         DRV_LOG(INFO, "Max number of segments: %u.",
 960                 (unsigned int)RTE_MIN(
 961                         MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
 962                         (uint16_t)(priv->max_rdmar_ds - 2)));
 963         pthread_mutex_lock(&priv_list_lock);
 964         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 965         pthread_mutex_unlock(&priv_list_lock);
 966
 967         rte_cryptodev_pmd_probing_finish(crypto_dev);
 968
 969         return 0;
 970 }
 971
 972 static int
 973 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
 974 {
 975         struct mlx5_crypto_priv *priv = NULL;
 976
 977         pthread_mutex_lock(&priv_list_lock);
 978         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 979                 if (priv->crypto_dev->device == cdev->dev)
 980                         break;
 981         if (priv)
 982                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 983         pthread_mutex_unlock(&priv_list_lock);
 984         if (priv) {
 985                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 986                 mlx5_devx_uar_release(&priv->uar);
 987                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 988         }
 989         return 0;
 990 }
 991
 992 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 993                 {
 994                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 995                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 996                 },
 997                 {
 998                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 999                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
1000                 },
1001                 {
1002                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
1003                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
1004                 },
1005                 {
1006                         .vendor_id = 0
1007                 }
1008 };
1009
1010 static struct mlx5_class_driver mlx5_crypto_driver = {
1011         .drv_class = MLX5_CLASS_CRYPTO,
1012         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
1013         .id_table = mlx5_crypto_pci_id_map,
1014         .probe = mlx5_crypto_dev_probe,
1015         .remove = mlx5_crypto_dev_remove,
1016 };
1017
1018 RTE_INIT(rte_mlx5_crypto_init)
1019 {
1020         pthread_mutex_init(&priv_list_lock, NULL);
1021         mlx5_common_init();
1022         if (mlx5_glue != NULL)
1023                 mlx5_class_driver_register(&mlx5_crypto_driver);
1024 }
1025
1026 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
1027                                mlx5_crypto_driver_id);
1028
1029 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
1030 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
1031 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
1032 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");