drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_eal_paging.h>
   8 #include <rte_errno.h>
   9 #include <rte_log.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_memory.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 128
  24 #define MLX5_CRYPTO_MAX_SEGS 56
  25
  26 #define MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode) \
  27         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  28          RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
  29          RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
  30          RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
  31          RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
  32          (wrapped_mode ? RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY : 0) | \
  33          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  34
  35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  36                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  37 static pthread_mutex_t priv_list_lock;
  38
  39 int mlx5_crypto_logtype;
  40
  41 uint8_t mlx5_crypto_driver_id;
  42
  43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  44         {               /* AES XTS */
  45                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  46                 {.sym = {
  47                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  48                         {.cipher = {
  49                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  50                                 .block_size = 16,
  51                                 .key_size = {
  52                                         .min = 32,
  53                                         .max = 64,
  54                                         .increment = 32
  55                                 },
  56                                 .iv_size = {
  57                                         .min = 16,
  58                                         .max = 16,
  59                                         .increment = 0
  60                                 },
  61                                 .dataunit_set =
  62                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  63                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES |
  64                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES,
  65                         }, }
  66                 }, }
  67         },
  68 };
  69
  70 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  71
  72 static const struct rte_driver mlx5_drv = {
  73         .name = mlx5_crypto_drv_name,
  74         .alias = mlx5_crypto_drv_name
  75 };
  76
  77 static struct cryptodev_driver mlx5_cryptodev_driver;
  78
  79 struct mlx5_crypto_session {
  80         uint32_t bs_bpt_eo_es;
  81         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  82          * saved in big endian format.
  83          */
  84         uint32_t bsp_res;
  85         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  86          * endian format.
  87          */
  88         uint32_t iv_offset:16;
  89         /**< Starting point for Initialisation Vector. */
  90         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  91         uint32_t dek_id; /**< DEK ID */
  92 } __rte_packed;
  93
  94 static void
  95 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  96                           struct rte_cryptodev_info *dev_info)
  97 {
  98         struct mlx5_crypto_priv *priv = dev->data->dev_private;
  99
 100         RTE_SET_USED(dev);
 101         if (dev_info != NULL) {
 102                 dev_info->driver_id = mlx5_crypto_driver_id;
 103                 dev_info->feature_flags =
 104                         MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode);
 105                 dev_info->capabilities = mlx5_crypto_caps;
 106                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 107                 dev_info->min_mbuf_headroom_req = 0;
 108                 dev_info->min_mbuf_tailroom_req = 0;
 109                 dev_info->sym.max_nb_sessions = 0;
 110                 /*
 111                  * If 0, the device does not have any limitation in number of
 112                  * sessions that can be used.
 113                  */
 114         }
 115 }
 116
 117 static int
 118 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 119                           struct rte_cryptodev_config *config)
 120 {
 121         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 122
 123         if (config == NULL) {
 124                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 125                 return -EINVAL;
 126         }
 127         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 128                 DRV_LOG(ERR,
 129                         "Disabled symmetric crypto feature is not supported.");
 130                 return -ENOTSUP;
 131         }
 132         if (mlx5_crypto_dek_setup(priv) != 0) {
 133                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 134                 return -ENOMEM;
 135         }
 136         priv->dev_config = *config;
 137         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 138         return 0;
 139 }
 140
 141 static void
 142 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 143 {
 144         RTE_SET_USED(dev);
 145 }
 146
 147 static int
 148 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 149 {
 150         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 151
 152         return mlx5_dev_mempool_subscribe(priv->cdev);
 153 }
 154
 155 static int
 156 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 157 {
 158         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 159
 160         mlx5_crypto_dek_unset(priv);
 161         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 162         return 0;
 163 }
 164
 165 static unsigned int
 166 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 167 {
 168         return sizeof(struct mlx5_crypto_session);
 169 }
 170
 171 static int
 172 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 173                                   struct rte_crypto_sym_xform *xform,
 174                                   struct rte_cryptodev_sym_session *session,
 175                                   struct rte_mempool *mp)
 176 {
 177         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 178         struct mlx5_crypto_session *sess_private_data;
 179         struct rte_crypto_cipher_xform *cipher;
 180         uint8_t encryption_order;
 181         int ret;
 182
 183         if (unlikely(xform->next != NULL)) {
 184                 DRV_LOG(ERR, "Xform next is not supported.");
 185                 return -ENOTSUP;
 186         }
 187         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 188                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 189                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 190                 return -ENOTSUP;
 191         }
 192         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 193         if (ret != 0) {
 194                 DRV_LOG(ERR,
 195                         "Failed to get session %p private data from mempool.",
 196                         sess_private_data);
 197                 return -ENOMEM;
 198         }
 199         cipher = &xform->cipher;
 200         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 201         if (sess_private_data->dek == NULL) {
 202                 rte_mempool_put(mp, sess_private_data);
 203                 DRV_LOG(ERR, "Failed to prepare dek.");
 204                 return -ENOMEM;
 205         }
 206         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 207                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 208         else
 209                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 210         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 211                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 212                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 213                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 214                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 215         switch (xform->cipher.dataunit_len) {
 216         case 0:
 217                 sess_private_data->bsp_res = 0;
 218                 break;
 219         case 512:
 220                 sess_private_data->bsp_res = rte_cpu_to_be_32
 221                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 222                                              MLX5_BLOCK_SIZE_OFFSET);
 223                 break;
 224         case 4096:
 225                 sess_private_data->bsp_res = rte_cpu_to_be_32
 226                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 227                                              MLX5_BLOCK_SIZE_OFFSET);
 228                 break;
 229         case 1048576:
 230                 sess_private_data->bsp_res = rte_cpu_to_be_32
 231                                              ((uint32_t)MLX5_BLOCK_SIZE_1MB <<
 232                                              MLX5_BLOCK_SIZE_OFFSET);
 233                 break;
 234         default:
 235                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 236                 return -ENOTSUP;
 237         }
 238         sess_private_data->iv_offset = cipher->iv.offset;
 239         sess_private_data->dek_id =
 240                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 241                                          0xffffff);
 242         set_sym_session_private_data(session, dev->driver_id,
 243                                      sess_private_data);
 244         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 245         return 0;
 246 }
 247
 248 static void
 249 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 250                               struct rte_cryptodev_sym_session *sess)
 251 {
 252         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 253         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 254                                                                 dev->driver_id);
 255
 256         if (unlikely(spriv == NULL)) {
 257                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 258                 return;
 259         }
 260         mlx5_crypto_dek_destroy(priv, spriv->dek);
 261         set_sym_session_private_data(sess, dev->driver_id, NULL);
 262         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 263         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 264 }
 265
 266 static void
 267 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n)
 268 {
 269         uint16_t i;
 270
 271         for (i = 0; i < n; i++)
 272                 if (qp->mkey[i])
 273                         claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
 274 }
 275
 276 static void
 277 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp)
 278 {
 279         if (qp == NULL)
 280                 return;
 281         mlx5_devx_qp_destroy(&qp->qp_obj);
 282         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 283         mlx5_devx_cq_destroy(&qp->cq_obj);
 284         rte_free(qp);
 285 }
 286
 287 static int
 288 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 289 {
 290         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 291
 292         mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
 293         mlx5_crypto_qp_release(qp);
 294         dev->data->queue_pairs[qp_id] = NULL;
 295         return 0;
 296 }
 297
 298 static __rte_noinline uint32_t
 299 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 300 {
 301         uint32_t bl = op->sym->cipher.data.length;
 302
 303         switch (bl) {
 304         case (1 << 20):
 305                 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
 306         case (1 << 12):
 307                 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
 308                                 MLX5_BLOCK_SIZE_OFFSET);
 309         case (1 << 9):
 310                 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
 311         default:
 312                 DRV_LOG(ERR, "Unknown block size: %u.", bl);
 313                 return UINT32_MAX;
 314         }
 315 }
 316
 317 static __rte_always_inline uint32_t
 318 mlx5_crypto_klm_set(struct mlx5_crypto_qp *qp, struct rte_mbuf *mbuf,
 319                     struct mlx5_wqe_dseg *klm, uint32_t offset,
 320                     uint32_t *remain)
 321 {
 322         uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 323         uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 324
 325         if (data_len > *remain)
 326                 data_len = *remain;
 327         *remain -= data_len;
 328         klm->bcount = rte_cpu_to_be_32(data_len);
 329         klm->pbuf = rte_cpu_to_be_64(addr);
 330         klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
 331         return klm->lkey;
 332
 333 }
 334
 335 static __rte_always_inline uint32_t
 336 mlx5_crypto_klms_set(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op,
 337                      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm)
 338 {
 339         uint32_t remain_len = op->sym->cipher.data.length;
 340         uint32_t nb_segs = mbuf->nb_segs;
 341         uint32_t klm_n = 1u;
 342
 343         /* First mbuf needs to take the cipher offset. */
 344         if (unlikely(mlx5_crypto_klm_set(qp, mbuf, klm,
 345                      op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 346                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 347                 return 0;
 348         }
 349         while (remain_len) {
 350                 nb_segs--;
 351                 mbuf = mbuf->next;
 352                 if (unlikely(mbuf == NULL || nb_segs == 0)) {
 353                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 354                         return 0;
 355                 }
 356                 if (unlikely(mlx5_crypto_klm_set(qp, mbuf, ++klm, 0,
 357                                                  &remain_len) == UINT32_MAX)) {
 358                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 359                         return 0;
 360                 }
 361                 klm_n++;
 362         }
 363         return klm_n;
 364 }
 365
 366 static __rte_always_inline int
 367 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 368                          struct mlx5_crypto_qp *qp,
 369                          struct rte_crypto_op *op,
 370                          struct mlx5_umr_wqe *umr)
 371 {
 372         struct mlx5_crypto_session *sess = get_sym_session_private_data
 373                                 (op->sym->session, mlx5_crypto_driver_id);
 374         struct mlx5_wqe_cseg *cseg = &umr->ctr;
 375         struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
 376         struct mlx5_wqe_dseg *klms = &umr->kseg[0];
 377         struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
 378                                       RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
 379         uint32_t ds;
 380         bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 381         /* Set UMR WQE. */
 382         uint32_t klm_n = mlx5_crypto_klms_set(qp, op,
 383                                    ipl ? op->sym->m_src : op->sym->m_dst, klms);
 384
 385         if (unlikely(klm_n == 0))
 386                 return 0;
 387         bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
 388         if (unlikely(!sess->bsp_res)) {
 389                 bsf->bsp_res = mlx5_crypto_get_block_size(op);
 390                 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
 391                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 392                         return 0;
 393                 }
 394         } else {
 395                 bsf->bsp_res = sess->bsp_res;
 396         }
 397         bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
 398         memcpy(bsf->xts_initial_tweak,
 399                rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
 400         bsf->res_dp = sess->dek_id;
 401         mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
 402         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
 403         qp->db_pi += priv->umr_wqe_stride;
 404         /* Set RDMA_WRITE WQE. */
 405         cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 406         klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 407         if (!ipl) {
 408                 klm_n = mlx5_crypto_klms_set(qp, op, op->sym->m_src, klms);
 409                 if (unlikely(klm_n == 0))
 410                         return 0;
 411         } else {
 412                 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
 413         }
 414         ds = 2 + klm_n;
 415         cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 416         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 417                                                         MLX5_OPCODE_RDMA_WRITE);
 418         ds = RTE_ALIGN(ds, 4);
 419         qp->db_pi += ds >> 2;
 420         /* Set NOP WQE if needed. */
 421         if (priv->max_rdmar_ds > ds) {
 422                 cseg += ds;
 423                 ds = priv->max_rdmar_ds - ds;
 424                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 425                 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 426                                                                MLX5_OPCODE_NOP);
 427                 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
 428         }
 429         qp->wqe = (uint8_t *)cseg;
 430         return 1;
 431 }
 432
 433 static uint16_t
 434 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 435                           uint16_t nb_ops)
 436 {
 437         struct mlx5_crypto_qp *qp = queue_pair;
 438         struct mlx5_crypto_priv *priv = qp->priv;
 439         struct mlx5_umr_wqe *umr;
 440         struct rte_crypto_op *op;
 441         uint16_t mask = qp->entries_n - 1;
 442         uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
 443         uint32_t idx;
 444
 445         if (remain < nb_ops)
 446                 nb_ops = remain;
 447         else
 448                 remain = nb_ops;
 449         if (unlikely(remain == 0))
 450                 return 0;
 451         do {
 452                 idx = qp->pi & mask;
 453                 op = *ops++;
 454                 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 455                         priv->wqe_set_size * idx);
 456                 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
 457                         qp->stats.enqueue_err_count++;
 458                         if (remain != nb_ops) {
 459                                 qp->stats.enqueued_count -= remain;
 460                                 break;
 461                         }
 462                         return 0;
 463                 }
 464                 qp->ops[idx] = op;
 465                 qp->pi++;
 466         } while (--remain);
 467         qp->stats.enqueued_count += nb_ops;
 468         mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)qp->wqe,
 469                            qp->db_pi, &qp->qp_obj.db_rec[MLX5_SND_DBR],
 470                            !priv->uar.dbnc);
 471         return nb_ops;
 472 }
 473
 474 static __rte_noinline void
 475 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 476 {
 477         const uint32_t idx = qp->ci & (qp->entries_n - 1);
 478         volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
 479                                                         &qp->cq_obj.cqes[idx];
 480
 481         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 482         qp->stats.dequeue_err_count++;
 483         DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 484 }
 485
 486 static uint16_t
 487 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 488                           uint16_t nb_ops)
 489 {
 490         struct mlx5_crypto_qp *qp = queue_pair;
 491         volatile struct mlx5_cqe *restrict cqe;
 492         struct rte_crypto_op *restrict op;
 493         const unsigned int cq_size = qp->entries_n;
 494         const unsigned int mask = cq_size - 1;
 495         uint32_t idx;
 496         uint32_t next_idx = qp->ci & mask;
 497         const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
 498         uint16_t i = 0;
 499         int ret;
 500
 501         if (unlikely(max == 0))
 502                 return 0;
 503         do {
 504                 idx = next_idx;
 505                 next_idx = (qp->ci + 1) & mask;
 506                 op = qp->ops[idx];
 507                 cqe = &qp->cq_obj.cqes[idx];
 508                 ret = check_cqe(cqe, cq_size, qp->ci);
 509                 rte_io_rmb();
 510                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 511                         if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
 512                                 mlx5_crypto_cqe_err_handle(qp, op);
 513                         break;
 514                 }
 515                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 516                 ops[i++] = op;
 517                 qp->ci++;
 518         } while (i < max);
 519         if (likely(i != 0)) {
 520                 rte_io_wmb();
 521                 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
 522                 qp->stats.dequeued_count += i;
 523         }
 524         return i;
 525 }
 526
 527 static void
 528 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 529 {
 530         uint32_t i;
 531
 532         for (i = 0 ; i < qp->entries_n; i++) {
 533                 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 534                         i * priv->wqe_set_size);
 535                 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
 536                                                                      (cseg + 1);
 537                 struct mlx5_wqe_umr_bsf_seg *bsf =
 538                         (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
 539                                                        priv->umr_wqe_size)) - 1;
 540                 struct mlx5_wqe_rseg *rseg;
 541
 542                 /* Init UMR WQE. */
 543                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
 544                                          (priv->umr_wqe_size / MLX5_WSEG_SIZE));
 545                 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 546                                        MLX5_COMP_MODE_OFFSET);
 547                 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
 548                 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
 549                 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
 550                 ucseg->ko_to_bs = rte_cpu_to_be_32
 551                         ((MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size) <<
 552                          MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
 553                 bsf->keytag = priv->keytag;
 554                 /* Init RDMA WRITE WQE. */
 555                 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 556                 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
 557                                       MLX5_COMP_MODE_OFFSET) |
 558                                       MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
 559                 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
 560                 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
 561         }
 562 }
 563
 564 static int
 565 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
 566                                   struct mlx5_crypto_qp *qp)
 567 {
 568         struct mlx5_umr_wqe *umr;
 569         uint32_t i;
 570         struct mlx5_devx_mkey_attr attr = {
 571                 .pd = priv->cdev->pdn,
 572                 .umr_en = 1,
 573                 .crypto_en = 1,
 574                 .set_remote_rw = 1,
 575                 .klm_num = MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
 576         };
 577
 578         for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
 579            i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
 580                 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
 581                 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
 582                 if (!qp->mkey[i])
 583                         goto error;
 584         }
 585         return 0;
 586 error:
 587         DRV_LOG(ERR, "Failed to allocate indirect mkey.");
 588         mlx5_crypto_indirect_mkeys_release(qp, i);
 589         return -1;
 590 }
 591
 592 static int
 593 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 594                              const struct rte_cryptodev_qp_conf *qp_conf,
 595                              int socket_id)
 596 {
 597         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 598         struct mlx5_devx_qp_attr attr = {0};
 599         struct mlx5_crypto_qp *qp;
 600         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 601         uint32_t ret;
 602         uint32_t alloc_size = sizeof(*qp);
 603         uint32_t log_wqbb_n;
 604         struct mlx5_devx_cq_attr cq_attr = {
 605                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar.obj),
 606         };
 607
 608         if (dev->data->queue_pairs[qp_id] != NULL)
 609                 mlx5_crypto_queue_pair_release(dev, qp_id);
 610         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 611         alloc_size += (sizeof(struct rte_crypto_op *) +
 612                        sizeof(struct mlx5_devx_obj *)) *
 613                        RTE_BIT32(log_nb_desc);
 614         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 615                                 socket_id);
 616         if (qp == NULL) {
 617                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 618                 rte_errno = ENOMEM;
 619                 return -rte_errno;
 620         }
 621         if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc,
 622                                 &cq_attr, socket_id) != 0) {
 623                 DRV_LOG(ERR, "Failed to create CQ.");
 624                 goto error;
 625         }
 626         log_wqbb_n = rte_log2_u32(RTE_BIT32(log_nb_desc) *
 627                                 (priv->wqe_set_size / MLX5_SEND_WQE_BB));
 628         attr.pd = priv->cdev->pdn;
 629         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar.obj);
 630         attr.cqn = qp->cq_obj.cq->id;
 631         attr.num_of_receive_wqes = 0;
 632         attr.num_of_send_wqbbs = RTE_BIT32(log_wqbb_n);
 633         attr.ts_format =
 634                 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 635         ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj,
 636                                         attr.num_of_send_wqbbs * MLX5_WQE_SIZE,
 637                                         &attr, socket_id);
 638         if (ret) {
 639                 DRV_LOG(ERR, "Failed to create QP.");
 640                 goto error;
 641         }
 642         if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
 643                               priv->dev_config.socket_id) != 0) {
 644                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 645                         (uint32_t)qp_id);
 646                 rte_errno = ENOMEM;
 647                 goto error;
 648         }
 649         /*
 650          * In Order to configure self loopback, when calling devx qp2rts the
 651          * remote QP id that is used is the id of the same QP.
 652          */
 653         if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id))
 654                 goto error;
 655         qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 656                                                            RTE_CACHE_LINE_SIZE);
 657         qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
 658         qp->entries_n = 1 << log_nb_desc;
 659         if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
 660                 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
 661                 rte_errno = ENOMEM;
 662                 goto error;
 663         }
 664         mlx5_crypto_qp_init(priv, qp);
 665         qp->priv = priv;
 666         dev->data->queue_pairs[qp_id] = qp;
 667         return 0;
 668 error:
 669         mlx5_crypto_qp_release(qp);
 670         return -1;
 671 }
 672
 673 static void
 674 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
 675                       struct rte_cryptodev_stats *stats)
 676 {
 677         int qp_id;
 678
 679         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 680                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 681
 682                 stats->enqueued_count += qp->stats.enqueued_count;
 683                 stats->dequeued_count += qp->stats.dequeued_count;
 684                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 685                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 686         }
 687 }
 688
 689 static void
 690 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
 691 {
 692         int qp_id;
 693
 694         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 695                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 696
 697                 memset(&qp->stats, 0, sizeof(qp->stats));
 698         }
 699 }
 700
 701 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 702         .dev_configure                  = mlx5_crypto_dev_configure,
 703         .dev_start                      = mlx5_crypto_dev_start,
 704         .dev_stop                       = mlx5_crypto_dev_stop,
 705         .dev_close                      = mlx5_crypto_dev_close,
 706         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 707         .stats_get                      = mlx5_crypto_stats_get,
 708         .stats_reset                    = mlx5_crypto_stats_reset,
 709         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 710         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 711         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 712         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 713         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 714         .sym_get_raw_dp_ctx_size        = NULL,
 715         .sym_configure_raw_dp_ctx       = NULL,
 716 };
 717
 718 static int
 719 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 720 {
 721         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 722         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 723         unsigned long tmp;
 724         FILE *file;
 725         int ret;
 726         int i;
 727
 728         if (strcmp(key, "wcs_file") == 0) {
 729                 file = fopen(val, "rb");
 730                 if (file == NULL) {
 731                         rte_errno = ENOTSUP;
 732                         return -rte_errno;
 733                 }
 734                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 735                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 736                         if (ret <= 0) {
 737                                 fclose(file);
 738                                 DRV_LOG(ERR,
 739                                         "Failed to read credential from file.");
 740                                 rte_errno = EINVAL;
 741                                 return -rte_errno;
 742                         }
 743                 }
 744                 fclose(file);
 745                 devarg_prms->login_devarg = true;
 746                 return 0;
 747         }
 748         errno = 0;
 749         tmp = strtoul(val, NULL, 0);
 750         if (errno) {
 751                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 752                 return -errno;
 753         }
 754         if (strcmp(key, "max_segs_num") == 0) {
 755                 if (!tmp) {
 756                         DRV_LOG(ERR, "max_segs_num must be greater than 0.");
 757                         rte_errno = EINVAL;
 758                         return -rte_errno;
 759                 }
 760                 devarg_prms->max_segs_num = (uint32_t)tmp;
 761         } else if (strcmp(key, "import_kek_id") == 0) {
 762                 attr->session_import_kek_ptr = (uint32_t)tmp;
 763         } else if (strcmp(key, "credential_id") == 0) {
 764                 attr->credential_pointer = (uint32_t)tmp;
 765         } else if (strcmp(key, "keytag") == 0) {
 766                 devarg_prms->keytag = tmp;
 767         }
 768         return 0;
 769 }
 770
 771 static int
 772 mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist,
 773                           struct mlx5_crypto_devarg_params *devarg_prms,
 774                           bool wrapped_mode)
 775 {
 776         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 777         const char **params = (const char *[]){
 778                 "credential_id",
 779                 "import_kek_id",
 780                 "keytag",
 781                 "max_segs_num",
 782                 "wcs_file",
 783                 NULL,
 784         };
 785
 786         /* Default values. */
 787         attr->credential_pointer = 0;
 788         attr->session_import_kek_ptr = 0;
 789         devarg_prms->keytag = 0;
 790         devarg_prms->max_segs_num = 8;
 791         if (mkvlist == NULL) {
 792                 if (!wrapped_mode)
 793                         return 0;
 794                 DRV_LOG(ERR,
 795                         "No login devargs in order to enable crypto operations in the device.");
 796                 rte_errno = EINVAL;
 797                 return -1;
 798         }
 799         if (mlx5_kvargs_process(mkvlist, params, mlx5_crypto_args_check_handler,
 800                                 devarg_prms) != 0) {
 801                 DRV_LOG(ERR, "Devargs handler function Failed.");
 802                 rte_errno = EINVAL;
 803                 return -1;
 804         }
 805         if (devarg_prms->login_devarg == false && wrapped_mode) {
 806                 DRV_LOG(ERR,
 807                         "No login credential devarg in order to enable crypto operations in the device while in wrapped import method.");
 808                 rte_errno = EINVAL;
 809                 return -1;
 810         }
 811         return 0;
 812 }
 813
 814 /*
 815  * Calculate UMR WQE size and RDMA Write WQE size with the
 816  * following limitations:
 817  *      - Each WQE size is multiple of 64.
 818  *      - The summarize of both UMR WQE and RDMA_W WQE is a power of 2.
 819  *      - The number of entries in the UMR WQE's KLM list is multiple of 4.
 820  */
 821 static void
 822 mlx5_crypto_get_wqe_sizes(uint32_t segs_num, uint32_t *umr_size,
 823                         uint32_t *rdmaw_size)
 824 {
 825         uint32_t diff, wqe_set_size;
 826
 827         *umr_size = MLX5_CRYPTO_UMR_WQE_STATIC_SIZE +
 828                         RTE_ALIGN(segs_num, 4) *
 829                         sizeof(struct mlx5_wqe_dseg);
 830         /* Make sure UMR WQE size is multiple of WQBB. */
 831         *umr_size = RTE_ALIGN(*umr_size, MLX5_SEND_WQE_BB);
 832         *rdmaw_size = sizeof(struct mlx5_rdma_write_wqe) +
 833                         sizeof(struct mlx5_wqe_dseg) *
 834                         (segs_num <= 2 ? 2 : 2 +
 835                         RTE_ALIGN(segs_num - 2, 4));
 836         /* Make sure RDMA_WRITE WQE size is multiple of WQBB. */
 837         *rdmaw_size = RTE_ALIGN(*rdmaw_size, MLX5_SEND_WQE_BB);
 838         wqe_set_size = *rdmaw_size + *umr_size;
 839         diff = rte_align32pow2(wqe_set_size) - wqe_set_size;
 840         /* Make sure wqe_set size is power of 2. */
 841         if (diff)
 842                 *umr_size += diff;
 843 }
 844
 845 static uint8_t
 846 mlx5_crypto_max_segs_num(uint16_t max_wqe_size)
 847 {
 848         int klms_sizes = max_wqe_size - MLX5_CRYPTO_UMR_WQE_STATIC_SIZE;
 849         uint32_t max_segs_cap = RTE_ALIGN_FLOOR(klms_sizes, MLX5_SEND_WQE_BB) /
 850                         sizeof(struct mlx5_wqe_dseg);
 851
 852         MLX5_ASSERT(klms_sizes >= MLX5_SEND_WQE_BB);
 853         while (max_segs_cap) {
 854                 uint32_t umr_wqe_size, rdmw_wqe_size;
 855
 856                 mlx5_crypto_get_wqe_sizes(max_segs_cap, &umr_wqe_size,
 857                                                 &rdmw_wqe_size);
 858                 if (umr_wqe_size <= max_wqe_size &&
 859                                 rdmw_wqe_size <= max_wqe_size)
 860                         break;
 861                 max_segs_cap -= 4;
 862         }
 863         return max_segs_cap;
 864 }
 865
 866 static int
 867 mlx5_crypto_configure_wqe_size(struct mlx5_crypto_priv *priv,
 868                                 uint16_t max_wqe_size, uint32_t max_segs_num)
 869 {
 870         uint32_t rdmw_wqe_size, umr_wqe_size;
 871
 872         mlx5_crypto_get_wqe_sizes(max_segs_num, &umr_wqe_size,
 873                                         &rdmw_wqe_size);
 874         priv->wqe_set_size = rdmw_wqe_size + umr_wqe_size;
 875         if (umr_wqe_size > max_wqe_size ||
 876                                 rdmw_wqe_size > max_wqe_size) {
 877                 DRV_LOG(ERR, "Invalid max_segs_num: %u. should be %u or lower.",
 878                         max_segs_num,
 879                         mlx5_crypto_max_segs_num(max_wqe_size));
 880                 rte_errno = EINVAL;
 881                 return -EINVAL;
 882         }
 883         priv->umr_wqe_size = (uint16_t)umr_wqe_size;
 884         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
 885         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 886         return 0;
 887 }
 888
 889 static int
 890 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 891                       struct mlx5_kvargs_ctrl *mkvlist)
 892 {
 893         struct rte_cryptodev *crypto_dev;
 894         struct mlx5_devx_obj *login;
 895         struct mlx5_crypto_priv *priv;
 896         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 897         struct rte_cryptodev_pmd_init_params init_params = {
 898                 .name = "",
 899                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 900                 .socket_id = cdev->dev->numa_node,
 901                 .max_nb_queue_pairs =
 902                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 903         };
 904         const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
 905         int ret;
 906         bool wrapped_mode;
 907
 908         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 909                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 910                 rte_errno = ENOTSUP;
 911                 return -rte_errno;
 912         }
 913         if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
 914                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 915                         "operations, maybe old FW/OFED version?");
 916                 rte_errno = ENOTSUP;
 917                 return -ENOTSUP;
 918         }
 919         wrapped_mode = !!cdev->config.hca_attr.crypto_wrapped_import_method;
 920         ret = mlx5_crypto_parse_devargs(mkvlist, &devarg_prms, wrapped_mode);
 921         if (ret) {
 922                 DRV_LOG(ERR, "Failed to parse devargs.");
 923                 return -rte_errno;
 924         }
 925         crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
 926                                               &init_params);
 927         if (crypto_dev == NULL) {
 928                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
 929                 return -ENODEV;
 930         }
 931         DRV_LOG(INFO,
 932                 "Crypto device %s was created successfully.", ibdev_name);
 933         crypto_dev->dev_ops = &mlx5_crypto_ops;
 934         crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 935         crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 936         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode);
 937         crypto_dev->driver_id = mlx5_crypto_driver_id;
 938         priv = crypto_dev->data->dev_private;
 939         priv->cdev = cdev;
 940         priv->crypto_dev = crypto_dev;
 941         priv->is_wrapped_mode = wrapped_mode;
 942         if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
 943                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 944                 return -1;
 945         }
 946         if (wrapped_mode) {
 947                 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
 948                                                       &devarg_prms.login_attr);
 949                 if (login == NULL) {
 950                         DRV_LOG(ERR, "Failed to configure login.");
 951                         mlx5_devx_uar_release(&priv->uar);
 952                         rte_cryptodev_pmd_destroy(priv->crypto_dev);
 953                         return -rte_errno;
 954                 }
 955                 priv->login_obj = login;
 956         }
 957         ret = mlx5_crypto_configure_wqe_size(priv,
 958                 cdev->config.hca_attr.max_wqe_sz_sq, devarg_prms.max_segs_num);
 959         if (ret) {
 960                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 961                 mlx5_devx_uar_release(&priv->uar);
 962                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 963                 return -1;
 964         }
 965         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 966         DRV_LOG(INFO, "Max number of segments: %u.",
 967                 (unsigned int)RTE_MIN(
 968                         MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
 969                         (uint16_t)(priv->max_rdmar_ds - 2)));
 970         pthread_mutex_lock(&priv_list_lock);
 971         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 972         pthread_mutex_unlock(&priv_list_lock);
 973
 974         rte_cryptodev_pmd_probing_finish(crypto_dev);
 975
 976         return 0;
 977 }
 978
 979 static int
 980 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
 981 {
 982         struct mlx5_crypto_priv *priv = NULL;
 983
 984         pthread_mutex_lock(&priv_list_lock);
 985         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 986                 if (priv->crypto_dev->device == cdev->dev)
 987                         break;
 988         if (priv)
 989                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 990         pthread_mutex_unlock(&priv_list_lock);
 991         if (priv) {
 992                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 993                 mlx5_devx_uar_release(&priv->uar);
 994                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 995         }
 996         return 0;
 997 }
 998
 999 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
1000                 {
1001                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
1002                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
1003                 },
1004                 {
1005                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
1006                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
1007                 },
1008                 {
1009                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
1010                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
1011                 },
1012                 {
1013                         .vendor_id = 0
1014                 }
1015 };
1016
1017 static struct mlx5_class_driver mlx5_crypto_driver = {
1018         .drv_class = MLX5_CLASS_CRYPTO,
1019         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
1020         .id_table = mlx5_crypto_pci_id_map,
1021         .probe = mlx5_crypto_dev_probe,
1022         .remove = mlx5_crypto_dev_remove,
1023 };
1024
1025 RTE_INIT(rte_mlx5_crypto_init)
1026 {
1027         pthread_mutex_init(&priv_list_lock, NULL);
1028         mlx5_common_init();
1029         if (mlx5_glue != NULL)
1030                 mlx5_class_driver_register(&mlx5_crypto_driver);
1031 }
1032
1033 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
1034                                mlx5_crypto_driver_id);
1035
1036 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
1037 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
1038 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
1039 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");