drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_eal_paging.h>
   8 #include <rte_errno.h>
   9 #include <rte_log.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_memory.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 1024
  24 #define MLX5_CRYPTO_MAX_SEGS 56
  25
  26 #define MLX5_CRYPTO_FEATURE_FLAGS \
  27         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  28          RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
  29          RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
  30          RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
  31          RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
  32          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  33          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  34
  35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  36                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  37 static pthread_mutex_t priv_list_lock;
  38
  39 int mlx5_crypto_logtype;
  40
  41 uint8_t mlx5_crypto_driver_id;
  42
  43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  44         {               /* AES XTS */
  45                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  46                 {.sym = {
  47                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  48                         {.cipher = {
  49                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  50                                 .block_size = 16,
  51                                 .key_size = {
  52                                         .min = 32,
  53                                         .max = 64,
  54                                         .increment = 32
  55                                 },
  56                                 .iv_size = {
  57                                         .min = 16,
  58                                         .max = 16,
  59                                         .increment = 0
  60                                 },
  61                                 .dataunit_set =
  62                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  63                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES |
  64                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES,
  65                         }, }
  66                 }, }
  67         },
  68 };
  69
  70 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  71
  72 static const struct rte_driver mlx5_drv = {
  73         .name = mlx5_crypto_drv_name,
  74         .alias = mlx5_crypto_drv_name
  75 };
  76
  77 static struct cryptodev_driver mlx5_cryptodev_driver;
  78
  79 struct mlx5_crypto_session {
  80         uint32_t bs_bpt_eo_es;
  81         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  82          * saved in big endian format.
  83          */
  84         uint32_t bsp_res;
  85         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  86          * endian format.
  87          */
  88         uint32_t iv_offset:16;
  89         /**< Starting point for Initialisation Vector. */
  90         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  91         uint32_t dek_id; /**< DEK ID */
  92 } __rte_packed;
  93
  94 static void
  95 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  96                           struct rte_cryptodev_info *dev_info)
  97 {
  98         RTE_SET_USED(dev);
  99         if (dev_info != NULL) {
 100                 dev_info->driver_id = mlx5_crypto_driver_id;
 101                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 102                 dev_info->capabilities = mlx5_crypto_caps;
 103                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 104                 dev_info->min_mbuf_headroom_req = 0;
 105                 dev_info->min_mbuf_tailroom_req = 0;
 106                 dev_info->sym.max_nb_sessions = 0;
 107                 /*
 108                  * If 0, the device does not have any limitation in number of
 109                  * sessions that can be used.
 110                  */
 111         }
 112 }
 113
 114 static int
 115 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 116                           struct rte_cryptodev_config *config)
 117 {
 118         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 119
 120         if (config == NULL) {
 121                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 122                 return -EINVAL;
 123         }
 124         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 125                 DRV_LOG(ERR,
 126                         "Disabled symmetric crypto feature is not supported.");
 127                 return -ENOTSUP;
 128         }
 129         if (mlx5_crypto_dek_setup(priv) != 0) {
 130                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 131                 return -ENOMEM;
 132         }
 133         priv->dev_config = *config;
 134         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 135         return 0;
 136 }
 137
 138 static void
 139 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 140 {
 141         RTE_SET_USED(dev);
 142 }
 143
 144 static int
 145 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 146 {
 147         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 148
 149         return mlx5_dev_mempool_subscribe(priv->cdev);
 150 }
 151
 152 static int
 153 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 154 {
 155         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 156
 157         mlx5_crypto_dek_unset(priv);
 158         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 159         return 0;
 160 }
 161
 162 static unsigned int
 163 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 164 {
 165         return sizeof(struct mlx5_crypto_session);
 166 }
 167
 168 static int
 169 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 170                                   struct rte_crypto_sym_xform *xform,
 171                                   struct rte_cryptodev_sym_session *session,
 172                                   struct rte_mempool *mp)
 173 {
 174         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 175         struct mlx5_crypto_session *sess_private_data;
 176         struct rte_crypto_cipher_xform *cipher;
 177         uint8_t encryption_order;
 178         int ret;
 179
 180         if (unlikely(xform->next != NULL)) {
 181                 DRV_LOG(ERR, "Xform next is not supported.");
 182                 return -ENOTSUP;
 183         }
 184         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 185                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 186                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 187                 return -ENOTSUP;
 188         }
 189         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 190         if (ret != 0) {
 191                 DRV_LOG(ERR,
 192                         "Failed to get session %p private data from mempool.",
 193                         sess_private_data);
 194                 return -ENOMEM;
 195         }
 196         cipher = &xform->cipher;
 197         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 198         if (sess_private_data->dek == NULL) {
 199                 rte_mempool_put(mp, sess_private_data);
 200                 DRV_LOG(ERR, "Failed to prepare dek.");
 201                 return -ENOMEM;
 202         }
 203         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 204                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 205         else
 206                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 207         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 208                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 209                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 210                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 211                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 212         switch (xform->cipher.dataunit_len) {
 213         case 0:
 214                 sess_private_data->bsp_res = 0;
 215                 break;
 216         case 512:
 217                 sess_private_data->bsp_res = rte_cpu_to_be_32
 218                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 219                                              MLX5_BLOCK_SIZE_OFFSET);
 220                 break;
 221         case 4096:
 222                 sess_private_data->bsp_res = rte_cpu_to_be_32
 223                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 224                                              MLX5_BLOCK_SIZE_OFFSET);
 225                 break;
 226         case 1048576:
 227                 sess_private_data->bsp_res = rte_cpu_to_be_32
 228                                              ((uint32_t)MLX5_BLOCK_SIZE_1MB <<
 229                                              MLX5_BLOCK_SIZE_OFFSET);
 230                 break;
 231         default:
 232                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 233                 return -ENOTSUP;
 234         }
 235         sess_private_data->iv_offset = cipher->iv.offset;
 236         sess_private_data->dek_id =
 237                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 238                                          0xffffff);
 239         set_sym_session_private_data(session, dev->driver_id,
 240                                      sess_private_data);
 241         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 242         return 0;
 243 }
 244
 245 static void
 246 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 247                               struct rte_cryptodev_sym_session *sess)
 248 {
 249         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 250         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 251                                                                 dev->driver_id);
 252
 253         if (unlikely(spriv == NULL)) {
 254                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 255                 return;
 256         }
 257         mlx5_crypto_dek_destroy(priv, spriv->dek);
 258         set_sym_session_private_data(sess, dev->driver_id, NULL);
 259         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 260         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 261 }
 262
 263 static void
 264 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n)
 265 {
 266         uint16_t i;
 267
 268         for (i = 0; i < n; i++)
 269                 if (qp->mkey[i])
 270                         claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
 271 }
 272
 273 static void
 274 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp)
 275 {
 276         if (qp == NULL)
 277                 return;
 278         mlx5_devx_qp_destroy(&qp->qp_obj);
 279         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 280         mlx5_devx_cq_destroy(&qp->cq_obj);
 281         rte_free(qp);
 282 }
 283
 284 static int
 285 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 286 {
 287         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 288
 289         mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
 290         mlx5_crypto_qp_release(qp);
 291         dev->data->queue_pairs[qp_id] = NULL;
 292         return 0;
 293 }
 294
 295 static __rte_noinline uint32_t
 296 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 297 {
 298         uint32_t bl = op->sym->cipher.data.length;
 299
 300         switch (bl) {
 301         case (1 << 20):
 302                 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
 303         case (1 << 12):
 304                 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
 305                                 MLX5_BLOCK_SIZE_OFFSET);
 306         case (1 << 9):
 307                 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
 308         default:
 309                 DRV_LOG(ERR, "Unknown block size: %u.", bl);
 310                 return UINT32_MAX;
 311         }
 312 }
 313
 314 static __rte_always_inline uint32_t
 315 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 316                       struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
 317                       uint32_t offset, uint32_t *remain)
 318 {
 319         uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 320         uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 321
 322         if (data_len > *remain)
 323                 data_len = *remain;
 324         *remain -= data_len;
 325         klm->bcount = rte_cpu_to_be_32(data_len);
 326         klm->pbuf = rte_cpu_to_be_64(addr);
 327         klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf);
 328         return klm->lkey;
 329
 330 }
 331
 332 static __rte_always_inline uint32_t
 333 mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 334                      struct rte_crypto_op *op, struct rte_mbuf *mbuf,
 335                      struct mlx5_wqe_dseg *klm)
 336 {
 337         uint32_t remain_len = op->sym->cipher.data.length;
 338         uint32_t nb_segs = mbuf->nb_segs;
 339         uint32_t klm_n = 1u;
 340
 341         /* First mbuf needs to take the cipher offset. */
 342         if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
 343                      op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 344                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 345                 return 0;
 346         }
 347         while (remain_len) {
 348                 nb_segs--;
 349                 mbuf = mbuf->next;
 350                 if (unlikely(mbuf == NULL || nb_segs == 0)) {
 351                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 352                         return 0;
 353                 }
 354                 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
 355                                                  &remain_len) == UINT32_MAX)) {
 356                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 357                         return 0;
 358                 }
 359                 klm_n++;
 360         }
 361         return klm_n;
 362 }
 363
 364 static __rte_always_inline int
 365 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 366                          struct mlx5_crypto_qp *qp,
 367                          struct rte_crypto_op *op,
 368                          struct mlx5_umr_wqe *umr)
 369 {
 370         struct mlx5_crypto_session *sess = get_sym_session_private_data
 371                                 (op->sym->session, mlx5_crypto_driver_id);
 372         struct mlx5_wqe_cseg *cseg = &umr->ctr;
 373         struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
 374         struct mlx5_wqe_dseg *klms = &umr->kseg[0];
 375         struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
 376                                       RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
 377         uint32_t ds;
 378         bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 379         /* Set UMR WQE. */
 380         uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
 381                                    ipl ? op->sym->m_src : op->sym->m_dst, klms);
 382
 383         if (unlikely(klm_n == 0))
 384                 return 0;
 385         bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
 386         if (unlikely(!sess->bsp_res)) {
 387                 bsf->bsp_res = mlx5_crypto_get_block_size(op);
 388                 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
 389                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 390                         return 0;
 391                 }
 392         } else {
 393                 bsf->bsp_res = sess->bsp_res;
 394         }
 395         bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
 396         memcpy(bsf->xts_initial_tweak,
 397                rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
 398         bsf->res_dp = sess->dek_id;
 399         mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
 400         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
 401         qp->db_pi += priv->umr_wqe_stride;
 402         /* Set RDMA_WRITE WQE. */
 403         cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 404         klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 405         if (!ipl) {
 406                 klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
 407                                              klms);
 408                 if (unlikely(klm_n == 0))
 409                         return 0;
 410         } else {
 411                 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
 412         }
 413         ds = 2 + klm_n;
 414         cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 415         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 416                                                         MLX5_OPCODE_RDMA_WRITE);
 417         ds = RTE_ALIGN(ds, 4);
 418         qp->db_pi += ds >> 2;
 419         /* Set NOP WQE if needed. */
 420         if (priv->max_rdmar_ds > ds) {
 421                 cseg += ds;
 422                 ds = priv->max_rdmar_ds - ds;
 423                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 424                 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 425                                                                MLX5_OPCODE_NOP);
 426                 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
 427         }
 428         qp->wqe = (uint8_t *)cseg;
 429         return 1;
 430 }
 431
 432 static __rte_always_inline void
 433 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
 434 {
 435 #ifdef RTE_ARCH_64
 436         *priv->uar_addr = val;
 437 #else /* !RTE_ARCH_64 */
 438         rte_spinlock_lock(&priv->uar32_sl);
 439         *(volatile uint32_t *)priv->uar_addr = val;
 440         rte_io_wmb();
 441         *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
 442         rte_spinlock_unlock(&priv->uar32_sl);
 443 #endif
 444 }
 445
 446 static uint16_t
 447 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 448                           uint16_t nb_ops)
 449 {
 450         struct mlx5_crypto_qp *qp = queue_pair;
 451         struct mlx5_crypto_priv *priv = qp->priv;
 452         struct mlx5_umr_wqe *umr;
 453         struct rte_crypto_op *op;
 454         uint16_t mask = qp->entries_n - 1;
 455         uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
 456         uint32_t idx;
 457
 458         if (remain < nb_ops)
 459                 nb_ops = remain;
 460         else
 461                 remain = nb_ops;
 462         if (unlikely(remain == 0))
 463                 return 0;
 464         do {
 465                 idx = qp->pi & mask;
 466                 op = *ops++;
 467                 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 468                         priv->wqe_set_size * idx);
 469                 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
 470                         qp->stats.enqueue_err_count++;
 471                         if (remain != nb_ops) {
 472                                 qp->stats.enqueued_count -= remain;
 473                                 break;
 474                         }
 475                         return 0;
 476                 }
 477                 qp->ops[idx] = op;
 478                 qp->pi++;
 479         } while (--remain);
 480         qp->stats.enqueued_count += nb_ops;
 481         rte_io_wmb();
 482         qp->qp_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 483         rte_wmb();
 484         mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
 485         rte_wmb();
 486         return nb_ops;
 487 }
 488
 489 static __rte_noinline void
 490 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 491 {
 492         const uint32_t idx = qp->ci & (qp->entries_n - 1);
 493         volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
 494                                                         &qp->cq_obj.cqes[idx];
 495
 496         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 497         qp->stats.dequeue_err_count++;
 498         DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 499 }
 500
 501 static uint16_t
 502 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 503                           uint16_t nb_ops)
 504 {
 505         struct mlx5_crypto_qp *qp = queue_pair;
 506         volatile struct mlx5_cqe *restrict cqe;
 507         struct rte_crypto_op *restrict op;
 508         const unsigned int cq_size = qp->entries_n;
 509         const unsigned int mask = cq_size - 1;
 510         uint32_t idx;
 511         uint32_t next_idx = qp->ci & mask;
 512         const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
 513         uint16_t i = 0;
 514         int ret;
 515
 516         if (unlikely(max == 0))
 517                 return 0;
 518         do {
 519                 idx = next_idx;
 520                 next_idx = (qp->ci + 1) & mask;
 521                 op = qp->ops[idx];
 522                 cqe = &qp->cq_obj.cqes[idx];
 523                 ret = check_cqe(cqe, cq_size, qp->ci);
 524                 rte_io_rmb();
 525                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 526                         if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
 527                                 mlx5_crypto_cqe_err_handle(qp, op);
 528                         break;
 529                 }
 530                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 531                 ops[i++] = op;
 532                 qp->ci++;
 533         } while (i < max);
 534         if (likely(i != 0)) {
 535                 rte_io_wmb();
 536                 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
 537                 qp->stats.dequeued_count += i;
 538         }
 539         return i;
 540 }
 541
 542 static void
 543 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 544 {
 545         uint32_t i;
 546
 547         for (i = 0 ; i < qp->entries_n; i++) {
 548                 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 549                         i * priv->wqe_set_size);
 550                 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
 551                                                                      (cseg + 1);
 552                 struct mlx5_wqe_umr_bsf_seg *bsf =
 553                         (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
 554                                                        priv->umr_wqe_size)) - 1;
 555                 struct mlx5_wqe_rseg *rseg;
 556
 557                 /* Init UMR WQE. */
 558                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
 559                                          (priv->umr_wqe_size / MLX5_WSEG_SIZE));
 560                 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 561                                        MLX5_COMP_MODE_OFFSET);
 562                 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
 563                 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
 564                 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
 565                 ucseg->ko_to_bs = rte_cpu_to_be_32
 566                         ((RTE_ALIGN(priv->max_segs_num, 4u) <<
 567                          MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
 568                 bsf->keytag = priv->keytag;
 569                 /* Init RDMA WRITE WQE. */
 570                 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 571                 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
 572                                       MLX5_COMP_MODE_OFFSET) |
 573                                       MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
 574                 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
 575                 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
 576         }
 577 }
 578
 579 static int
 580 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
 581                                   struct mlx5_crypto_qp *qp)
 582 {
 583         struct mlx5_umr_wqe *umr;
 584         uint32_t i;
 585         struct mlx5_devx_mkey_attr attr = {
 586                 .pd = priv->cdev->pdn,
 587                 .umr_en = 1,
 588                 .crypto_en = 1,
 589                 .set_remote_rw = 1,
 590                 .klm_num = RTE_ALIGN(priv->max_segs_num, 4),
 591         };
 592
 593         for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
 594            i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
 595                 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
 596                 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
 597                 if (!qp->mkey[i])
 598                         goto error;
 599         }
 600         return 0;
 601 error:
 602         DRV_LOG(ERR, "Failed to allocate indirect mkey.");
 603         mlx5_crypto_indirect_mkeys_release(qp, i);
 604         return -1;
 605 }
 606
 607 static int
 608 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 609                              const struct rte_cryptodev_qp_conf *qp_conf,
 610                              int socket_id)
 611 {
 612         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 613         struct mlx5_devx_qp_attr attr = {0};
 614         struct mlx5_crypto_qp *qp;
 615         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 616         uint32_t ret;
 617         uint32_t alloc_size = sizeof(*qp);
 618         struct mlx5_devx_cq_attr cq_attr = {
 619                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
 620         };
 621
 622         if (dev->data->queue_pairs[qp_id] != NULL)
 623                 mlx5_crypto_queue_pair_release(dev, qp_id);
 624         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 625         alloc_size += (sizeof(struct rte_crypto_op *) +
 626                        sizeof(struct mlx5_devx_obj *)) *
 627                        RTE_BIT32(log_nb_desc);
 628         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 629                                 socket_id);
 630         if (qp == NULL) {
 631                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 632                 rte_errno = ENOMEM;
 633                 return -rte_errno;
 634         }
 635         if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc,
 636                                 &cq_attr, socket_id) != 0) {
 637                 DRV_LOG(ERR, "Failed to create CQ.");
 638                 goto error;
 639         }
 640         attr.pd = priv->cdev->pdn;
 641         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 642         attr.cqn = qp->cq_obj.cq->id;
 643         attr.rq_size = 0;
 644         attr.sq_size = RTE_BIT32(log_nb_desc);
 645         attr.ts_format =
 646                 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 647         ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, log_nb_desc,
 648                                   &attr, socket_id);
 649         if (ret) {
 650                 DRV_LOG(ERR, "Failed to create QP.");
 651                 goto error;
 652         }
 653         if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
 654                               priv->dev_config.socket_id) != 0) {
 655                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 656                         (uint32_t)qp_id);
 657                 rte_errno = ENOMEM;
 658                 goto error;
 659         }
 660         /*
 661          * In Order to configure self loopback, when calling devx qp2rts the
 662          * remote QP id that is used is the id of the same QP.
 663          */
 664         if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id))
 665                 goto error;
 666         qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 667                                                            RTE_CACHE_LINE_SIZE);
 668         qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
 669         qp->entries_n = 1 << log_nb_desc;
 670         if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
 671                 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
 672                 rte_errno = ENOMEM;
 673                 goto error;
 674         }
 675         mlx5_crypto_qp_init(priv, qp);
 676         qp->priv = priv;
 677         dev->data->queue_pairs[qp_id] = qp;
 678         return 0;
 679 error:
 680         mlx5_crypto_qp_release(qp);
 681         return -1;
 682 }
 683
 684 static void
 685 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
 686                       struct rte_cryptodev_stats *stats)
 687 {
 688         int qp_id;
 689
 690         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 691                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 692
 693                 stats->enqueued_count += qp->stats.enqueued_count;
 694                 stats->dequeued_count += qp->stats.dequeued_count;
 695                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 696                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 697         }
 698 }
 699
 700 static void
 701 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
 702 {
 703         int qp_id;
 704
 705         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 706                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 707
 708                 memset(&qp->stats, 0, sizeof(qp->stats));
 709         }
 710 }
 711
 712 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 713         .dev_configure                  = mlx5_crypto_dev_configure,
 714         .dev_start                      = mlx5_crypto_dev_start,
 715         .dev_stop                       = mlx5_crypto_dev_stop,
 716         .dev_close                      = mlx5_crypto_dev_close,
 717         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 718         .stats_get                      = mlx5_crypto_stats_get,
 719         .stats_reset                    = mlx5_crypto_stats_reset,
 720         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 721         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 722         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 723         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 724         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 725         .sym_get_raw_dp_ctx_size        = NULL,
 726         .sym_configure_raw_dp_ctx       = NULL,
 727 };
 728
 729 static void
 730 mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv)
 731 {
 732         if (priv->uar != NULL) {
 733                 mlx5_glue->devx_free_uar(priv->uar);
 734                 priv->uar = NULL;
 735         }
 736 }
 737
 738 static int
 739 mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv)
 740 {
 741         priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
 742         if (priv->uar)
 743                 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
 744         if (priv->uar == NULL || priv->uar_addr == NULL) {
 745                 rte_errno = errno;
 746                 DRV_LOG(ERR, "Failed to allocate UAR.");
 747                 return -1;
 748         }
 749         return 0;
 750 }
 751
 752
 753 static int
 754 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 755 {
 756         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 757         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 758         unsigned long tmp;
 759         FILE *file;
 760         int ret;
 761         int i;
 762
 763         if (strcmp(key, "class") == 0)
 764                 return 0;
 765         if (strcmp(key, "wcs_file") == 0) {
 766                 file = fopen(val, "rb");
 767                 if (file == NULL) {
 768                         rte_errno = ENOTSUP;
 769                         return -rte_errno;
 770                 }
 771                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 772                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 773                         if (ret <= 0) {
 774                                 fclose(file);
 775                                 DRV_LOG(ERR,
 776                                         "Failed to read credential from file.");
 777                                 rte_errno = EINVAL;
 778                                 return -rte_errno;
 779                         }
 780                 }
 781                 fclose(file);
 782                 devarg_prms->login_devarg = true;
 783                 return 0;
 784         }
 785         errno = 0;
 786         tmp = strtoul(val, NULL, 0);
 787         if (errno) {
 788                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 789                 return -errno;
 790         }
 791         if (strcmp(key, "max_segs_num") == 0) {
 792                 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
 793                         DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
 794                                 " be less than %d.",
 795                                 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
 796                         rte_errno = EINVAL;
 797                         return -rte_errno;
 798                 }
 799                 devarg_prms->max_segs_num = (uint32_t)tmp;
 800         } else if (strcmp(key, "import_kek_id") == 0) {
 801                 attr->session_import_kek_ptr = (uint32_t)tmp;
 802         } else if (strcmp(key, "credential_id") == 0) {
 803                 attr->credential_pointer = (uint32_t)tmp;
 804         } else if (strcmp(key, "keytag") == 0) {
 805                 devarg_prms->keytag = tmp;
 806         } else {
 807                 DRV_LOG(WARNING, "Invalid key %s.", key);
 808         }
 809         return 0;
 810 }
 811
 812 static int
 813 mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 814                           struct mlx5_crypto_devarg_params *devarg_prms)
 815 {
 816         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 817         struct rte_kvargs *kvlist;
 818
 819         /* Default values. */
 820         attr->credential_pointer = 0;
 821         attr->session_import_kek_ptr = 0;
 822         devarg_prms->keytag = 0;
 823         devarg_prms->max_segs_num = 8;
 824         if (devargs == NULL) {
 825                 DRV_LOG(ERR,
 826         "No login devargs in order to enable crypto operations in the device.");
 827                 rte_errno = EINVAL;
 828                 return -1;
 829         }
 830         kvlist = rte_kvargs_parse(devargs->args, NULL);
 831         if (kvlist == NULL) {
 832                 DRV_LOG(ERR, "Failed to parse devargs.");
 833                 rte_errno = EINVAL;
 834                 return -1;
 835         }
 836         if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
 837                            devarg_prms) != 0) {
 838                 DRV_LOG(ERR, "Devargs handler function Failed.");
 839                 rte_kvargs_free(kvlist);
 840                 rte_errno = EINVAL;
 841                 return -1;
 842         }
 843         rte_kvargs_free(kvlist);
 844         if (devarg_prms->login_devarg == false) {
 845                 DRV_LOG(ERR,
 846         "No login credential devarg in order to enable crypto operations "
 847         "in the device.");
 848                 rte_errno = EINVAL;
 849                 return -1;
 850         }
 851         return 0;
 852 }
 853
 854 static int
 855 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 856 {
 857         struct rte_cryptodev *crypto_dev;
 858         struct mlx5_devx_obj *login;
 859         struct mlx5_crypto_priv *priv;
 860         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 861         struct rte_cryptodev_pmd_init_params init_params = {
 862                 .name = "",
 863                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 864                 .socket_id = cdev->dev->numa_node,
 865                 .max_nb_queue_pairs =
 866                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 867         };
 868         const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
 869         uint16_t rdmw_wqe_size;
 870         int ret;
 871
 872         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 873                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 874                 rte_errno = ENOTSUP;
 875                 return -rte_errno;
 876         }
 877         if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
 878                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 879                         "operations, maybe old FW/OFED version?");
 880                 rte_errno = ENOTSUP;
 881                 return -ENOTSUP;
 882         }
 883         ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms);
 884         if (ret) {
 885                 DRV_LOG(ERR, "Failed to parse devargs.");
 886                 return -rte_errno;
 887         }
 888         login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
 889                                                       &devarg_prms.login_attr);
 890         if (login == NULL) {
 891                 DRV_LOG(ERR, "Failed to configure login.");
 892                 return -rte_errno;
 893         }
 894         crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
 895                                               &init_params);
 896         if (crypto_dev == NULL) {
 897                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
 898                 return -ENODEV;
 899         }
 900         DRV_LOG(INFO,
 901                 "Crypto device %s was created successfully.", ibdev_name);
 902         crypto_dev->dev_ops = &mlx5_crypto_ops;
 903         crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 904         crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 905         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 906         crypto_dev->driver_id = mlx5_crypto_driver_id;
 907         priv = crypto_dev->data->dev_private;
 908         priv->cdev = cdev;
 909         priv->login_obj = login;
 910         priv->crypto_dev = crypto_dev;
 911         if (mlx5_crypto_uar_prepare(priv) != 0) {
 912                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 913                 return -1;
 914         }
 915         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 916         priv->max_segs_num = devarg_prms.max_segs_num;
 917         priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
 918                              sizeof(struct mlx5_wqe_cseg) +
 919                              sizeof(struct mlx5_wqe_umr_cseg) +
 920                              sizeof(struct mlx5_wqe_mkey_cseg) +
 921                              RTE_ALIGN(priv->max_segs_num, 4) *
 922                              sizeof(struct mlx5_wqe_dseg);
 923         rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
 924                               sizeof(struct mlx5_wqe_dseg) *
 925                               (priv->max_segs_num <= 2 ? 2 : 2 +
 926                                RTE_ALIGN(priv->max_segs_num - 2, 4));
 927         priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
 928         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
 929         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 930         pthread_mutex_lock(&priv_list_lock);
 931         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 932         pthread_mutex_unlock(&priv_list_lock);
 933
 934         rte_cryptodev_pmd_probing_finish(crypto_dev);
 935
 936         return 0;
 937 }
 938
 939 static int
 940 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
 941 {
 942         struct mlx5_crypto_priv *priv = NULL;
 943
 944         pthread_mutex_lock(&priv_list_lock);
 945         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 946                 if (priv->crypto_dev->device == cdev->dev)
 947                         break;
 948         if (priv)
 949                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 950         pthread_mutex_unlock(&priv_list_lock);
 951         if (priv) {
 952                 mlx5_crypto_uar_release(priv);
 953                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 954                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 955         }
 956         return 0;
 957 }
 958
 959 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 960                 {
 961                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 962                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 963                 },
 964                 {
 965                         .vendor_id = 0
 966                 }
 967 };
 968
 969 static struct mlx5_class_driver mlx5_crypto_driver = {
 970         .drv_class = MLX5_CLASS_CRYPTO,
 971         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
 972         .id_table = mlx5_crypto_pci_id_map,
 973         .probe = mlx5_crypto_dev_probe,
 974         .remove = mlx5_crypto_dev_remove,
 975 };
 976
 977 RTE_INIT(rte_mlx5_crypto_init)
 978 {
 979         pthread_mutex_init(&priv_list_lock, NULL);
 980         mlx5_common_init();
 981         if (mlx5_glue != NULL)
 982                 mlx5_class_driver_register(&mlx5_crypto_driver);
 983 }
 984
 985 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
 986                                mlx5_crypto_driver_id);
 987
 988 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
 989 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
 990 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
 991 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");