drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_eal_paging.h>
   8 #include <rte_errno.h>
   9 #include <rte_log.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_memory.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 1024
  24 #define MLX5_CRYPTO_MAX_SEGS 56
  25
  26 #define MLX5_CRYPTO_FEATURE_FLAGS \
  27         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  28          RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
  29          RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
  30          RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
  31          RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
  32          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  33          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  34
  35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  36                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  37 static pthread_mutex_t priv_list_lock;
  38
  39 int mlx5_crypto_logtype;
  40
  41 uint8_t mlx5_crypto_driver_id;
  42
  43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  44         {               /* AES XTS */
  45                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  46                 {.sym = {
  47                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  48                         {.cipher = {
  49                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  50                                 .block_size = 16,
  51                                 .key_size = {
  52                                         .min = 32,
  53                                         .max = 64,
  54                                         .increment = 32
  55                                 },
  56                                 .iv_size = {
  57                                         .min = 16,
  58                                         .max = 16,
  59                                         .increment = 0
  60                                 },
  61                                 .dataunit_set =
  62                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  63                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES |
  64                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES,
  65                         }, }
  66                 }, }
  67         },
  68 };
  69
  70 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  71
  72 static const struct rte_driver mlx5_drv = {
  73         .name = mlx5_crypto_drv_name,
  74         .alias = mlx5_crypto_drv_name
  75 };
  76
  77 static struct cryptodev_driver mlx5_cryptodev_driver;
  78
  79 struct mlx5_crypto_session {
  80         uint32_t bs_bpt_eo_es;
  81         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  82          * saved in big endian format.
  83          */
  84         uint32_t bsp_res;
  85         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  86          * endian format.
  87          */
  88         uint32_t iv_offset:16;
  89         /**< Starting point for Initialisation Vector. */
  90         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  91         uint32_t dek_id; /**< DEK ID */
  92 } __rte_packed;
  93
  94 static void
  95 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  96                           struct rte_cryptodev_info *dev_info)
  97 {
  98         RTE_SET_USED(dev);
  99         if (dev_info != NULL) {
 100                 dev_info->driver_id = mlx5_crypto_driver_id;
 101                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 102                 dev_info->capabilities = mlx5_crypto_caps;
 103                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 104                 dev_info->min_mbuf_headroom_req = 0;
 105                 dev_info->min_mbuf_tailroom_req = 0;
 106                 dev_info->sym.max_nb_sessions = 0;
 107                 /*
 108                  * If 0, the device does not have any limitation in number of
 109                  * sessions that can be used.
 110                  */
 111         }
 112 }
 113
 114 static int
 115 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 116                           struct rte_cryptodev_config *config)
 117 {
 118         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 119
 120         if (config == NULL) {
 121                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 122                 return -EINVAL;
 123         }
 124         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 125                 DRV_LOG(ERR,
 126                         "Disabled symmetric crypto feature is not supported.");
 127                 return -ENOTSUP;
 128         }
 129         if (mlx5_crypto_dek_setup(priv) != 0) {
 130                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 131                 return -ENOMEM;
 132         }
 133         priv->dev_config = *config;
 134         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 135         return 0;
 136 }
 137
 138 static void
 139 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 140 {
 141         RTE_SET_USED(dev);
 142 }
 143
 144 static int
 145 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 146 {
 147         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 148
 149         return mlx5_dev_mempool_subscribe(priv->cdev);
 150 }
 151
 152 static int
 153 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 154 {
 155         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 156
 157         mlx5_crypto_dek_unset(priv);
 158         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 159         return 0;
 160 }
 161
 162 static unsigned int
 163 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 164 {
 165         return sizeof(struct mlx5_crypto_session);
 166 }
 167
 168 static int
 169 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 170                                   struct rte_crypto_sym_xform *xform,
 171                                   struct rte_cryptodev_sym_session *session,
 172                                   struct rte_mempool *mp)
 173 {
 174         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 175         struct mlx5_crypto_session *sess_private_data;
 176         struct rte_crypto_cipher_xform *cipher;
 177         uint8_t encryption_order;
 178         int ret;
 179
 180         if (unlikely(xform->next != NULL)) {
 181                 DRV_LOG(ERR, "Xform next is not supported.");
 182                 return -ENOTSUP;
 183         }
 184         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 185                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 186                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 187                 return -ENOTSUP;
 188         }
 189         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 190         if (ret != 0) {
 191                 DRV_LOG(ERR,
 192                         "Failed to get session %p private data from mempool.",
 193                         sess_private_data);
 194                 return -ENOMEM;
 195         }
 196         cipher = &xform->cipher;
 197         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 198         if (sess_private_data->dek == NULL) {
 199                 rte_mempool_put(mp, sess_private_data);
 200                 DRV_LOG(ERR, "Failed to prepare dek.");
 201                 return -ENOMEM;
 202         }
 203         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 204                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 205         else
 206                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 207         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 208                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 209                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 210                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 211                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 212         switch (xform->cipher.dataunit_len) {
 213         case 0:
 214                 sess_private_data->bsp_res = 0;
 215                 break;
 216         case 512:
 217                 sess_private_data->bsp_res = rte_cpu_to_be_32
 218                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 219                                              MLX5_BLOCK_SIZE_OFFSET);
 220                 break;
 221         case 4096:
 222                 sess_private_data->bsp_res = rte_cpu_to_be_32
 223                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 224                                              MLX5_BLOCK_SIZE_OFFSET);
 225                 break;
 226         case 1048576:
 227                 sess_private_data->bsp_res = rte_cpu_to_be_32
 228                                              ((uint32_t)MLX5_BLOCK_SIZE_1MB <<
 229                                              MLX5_BLOCK_SIZE_OFFSET);
 230                 break;
 231         default:
 232                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 233                 return -ENOTSUP;
 234         }
 235         sess_private_data->iv_offset = cipher->iv.offset;
 236         sess_private_data->dek_id =
 237                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 238                                          0xffffff);
 239         set_sym_session_private_data(session, dev->driver_id,
 240                                      sess_private_data);
 241         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 242         return 0;
 243 }
 244
 245 static void
 246 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 247                               struct rte_cryptodev_sym_session *sess)
 248 {
 249         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 250         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 251                                                                 dev->driver_id);
 252
 253         if (unlikely(spriv == NULL)) {
 254                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 255                 return;
 256         }
 257         mlx5_crypto_dek_destroy(priv, spriv->dek);
 258         set_sym_session_private_data(sess, dev->driver_id, NULL);
 259         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 260         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 261 }
 262
 263 static void
 264 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n)
 265 {
 266         uint16_t i;
 267
 268         for (i = 0; i < n; i++)
 269                 if (qp->mkey[i])
 270                         claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
 271 }
 272
 273 static void
 274 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp)
 275 {
 276         if (qp == NULL)
 277                 return;
 278         mlx5_devx_qp_destroy(&qp->qp_obj);
 279         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 280         mlx5_devx_cq_destroy(&qp->cq_obj);
 281         rte_free(qp);
 282 }
 283
 284 static int
 285 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 286 {
 287         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 288
 289         mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
 290         mlx5_crypto_qp_release(qp);
 291         dev->data->queue_pairs[qp_id] = NULL;
 292         return 0;
 293 }
 294
 295 static __rte_noinline uint32_t
 296 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 297 {
 298         uint32_t bl = op->sym->cipher.data.length;
 299
 300         switch (bl) {
 301         case (1 << 20):
 302                 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
 303         case (1 << 12):
 304                 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
 305                                 MLX5_BLOCK_SIZE_OFFSET);
 306         case (1 << 9):
 307                 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
 308         default:
 309                 DRV_LOG(ERR, "Unknown block size: %u.", bl);
 310                 return UINT32_MAX;
 311         }
 312 }
 313
 314 static __rte_always_inline uint32_t
 315 mlx5_crypto_klm_set(struct mlx5_crypto_qp *qp, struct rte_mbuf *mbuf,
 316                     struct mlx5_wqe_dseg *klm, uint32_t offset,
 317                     uint32_t *remain)
 318 {
 319         uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 320         uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 321
 322         if (data_len > *remain)
 323                 data_len = *remain;
 324         *remain -= data_len;
 325         klm->bcount = rte_cpu_to_be_32(data_len);
 326         klm->pbuf = rte_cpu_to_be_64(addr);
 327         klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
 328         return klm->lkey;
 329
 330 }
 331
 332 static __rte_always_inline uint32_t
 333 mlx5_crypto_klms_set(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op,
 334                      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm)
 335 {
 336         uint32_t remain_len = op->sym->cipher.data.length;
 337         uint32_t nb_segs = mbuf->nb_segs;
 338         uint32_t klm_n = 1u;
 339
 340         /* First mbuf needs to take the cipher offset. */
 341         if (unlikely(mlx5_crypto_klm_set(qp, mbuf, klm,
 342                      op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 343                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 344                 return 0;
 345         }
 346         while (remain_len) {
 347                 nb_segs--;
 348                 mbuf = mbuf->next;
 349                 if (unlikely(mbuf == NULL || nb_segs == 0)) {
 350                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 351                         return 0;
 352                 }
 353                 if (unlikely(mlx5_crypto_klm_set(qp, mbuf, ++klm, 0,
 354                                                  &remain_len) == UINT32_MAX)) {
 355                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 356                         return 0;
 357                 }
 358                 klm_n++;
 359         }
 360         return klm_n;
 361 }
 362
 363 static __rte_always_inline int
 364 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 365                          struct mlx5_crypto_qp *qp,
 366                          struct rte_crypto_op *op,
 367                          struct mlx5_umr_wqe *umr)
 368 {
 369         struct mlx5_crypto_session *sess = get_sym_session_private_data
 370                                 (op->sym->session, mlx5_crypto_driver_id);
 371         struct mlx5_wqe_cseg *cseg = &umr->ctr;
 372         struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
 373         struct mlx5_wqe_dseg *klms = &umr->kseg[0];
 374         struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
 375                                       RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
 376         uint32_t ds;
 377         bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 378         /* Set UMR WQE. */
 379         uint32_t klm_n = mlx5_crypto_klms_set(qp, op,
 380                                    ipl ? op->sym->m_src : op->sym->m_dst, klms);
 381
 382         if (unlikely(klm_n == 0))
 383                 return 0;
 384         bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
 385         if (unlikely(!sess->bsp_res)) {
 386                 bsf->bsp_res = mlx5_crypto_get_block_size(op);
 387                 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
 388                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 389                         return 0;
 390                 }
 391         } else {
 392                 bsf->bsp_res = sess->bsp_res;
 393         }
 394         bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
 395         memcpy(bsf->xts_initial_tweak,
 396                rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
 397         bsf->res_dp = sess->dek_id;
 398         mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
 399         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
 400         qp->db_pi += priv->umr_wqe_stride;
 401         /* Set RDMA_WRITE WQE. */
 402         cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 403         klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 404         if (!ipl) {
 405                 klm_n = mlx5_crypto_klms_set(qp, op, op->sym->m_src, klms);
 406                 if (unlikely(klm_n == 0))
 407                         return 0;
 408         } else {
 409                 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
 410         }
 411         ds = 2 + klm_n;
 412         cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 413         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 414                                                         MLX5_OPCODE_RDMA_WRITE);
 415         ds = RTE_ALIGN(ds, 4);
 416         qp->db_pi += ds >> 2;
 417         /* Set NOP WQE if needed. */
 418         if (priv->max_rdmar_ds > ds) {
 419                 cseg += ds;
 420                 ds = priv->max_rdmar_ds - ds;
 421                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 422                 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 423                                                                MLX5_OPCODE_NOP);
 424                 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
 425         }
 426         qp->wqe = (uint8_t *)cseg;
 427         return 1;
 428 }
 429
 430 static __rte_always_inline void
 431 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
 432 {
 433 #ifdef RTE_ARCH_64
 434         *priv->uar_addr = val;
 435 #else /* !RTE_ARCH_64 */
 436         rte_spinlock_lock(&priv->uar32_sl);
 437         *(volatile uint32_t *)priv->uar_addr = val;
 438         rte_io_wmb();
 439         *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
 440         rte_spinlock_unlock(&priv->uar32_sl);
 441 #endif
 442 }
 443
 444 static uint16_t
 445 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 446                           uint16_t nb_ops)
 447 {
 448         struct mlx5_crypto_qp *qp = queue_pair;
 449         struct mlx5_crypto_priv *priv = qp->priv;
 450         struct mlx5_umr_wqe *umr;
 451         struct rte_crypto_op *op;
 452         uint16_t mask = qp->entries_n - 1;
 453         uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
 454         uint32_t idx;
 455
 456         if (remain < nb_ops)
 457                 nb_ops = remain;
 458         else
 459                 remain = nb_ops;
 460         if (unlikely(remain == 0))
 461                 return 0;
 462         do {
 463                 idx = qp->pi & mask;
 464                 op = *ops++;
 465                 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 466                         priv->wqe_set_size * idx);
 467                 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
 468                         qp->stats.enqueue_err_count++;
 469                         if (remain != nb_ops) {
 470                                 qp->stats.enqueued_count -= remain;
 471                                 break;
 472                         }
 473                         return 0;
 474                 }
 475                 qp->ops[idx] = op;
 476                 qp->pi++;
 477         } while (--remain);
 478         qp->stats.enqueued_count += nb_ops;
 479         rte_io_wmb();
 480         qp->qp_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 481         rte_wmb();
 482         mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
 483         rte_wmb();
 484         return nb_ops;
 485 }
 486
 487 static __rte_noinline void
 488 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 489 {
 490         const uint32_t idx = qp->ci & (qp->entries_n - 1);
 491         volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
 492                                                         &qp->cq_obj.cqes[idx];
 493
 494         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 495         qp->stats.dequeue_err_count++;
 496         DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 497 }
 498
 499 static uint16_t
 500 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 501                           uint16_t nb_ops)
 502 {
 503         struct mlx5_crypto_qp *qp = queue_pair;
 504         volatile struct mlx5_cqe *restrict cqe;
 505         struct rte_crypto_op *restrict op;
 506         const unsigned int cq_size = qp->entries_n;
 507         const unsigned int mask = cq_size - 1;
 508         uint32_t idx;
 509         uint32_t next_idx = qp->ci & mask;
 510         const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
 511         uint16_t i = 0;
 512         int ret;
 513
 514         if (unlikely(max == 0))
 515                 return 0;
 516         do {
 517                 idx = next_idx;
 518                 next_idx = (qp->ci + 1) & mask;
 519                 op = qp->ops[idx];
 520                 cqe = &qp->cq_obj.cqes[idx];
 521                 ret = check_cqe(cqe, cq_size, qp->ci);
 522                 rte_io_rmb();
 523                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 524                         if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
 525                                 mlx5_crypto_cqe_err_handle(qp, op);
 526                         break;
 527                 }
 528                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 529                 ops[i++] = op;
 530                 qp->ci++;
 531         } while (i < max);
 532         if (likely(i != 0)) {
 533                 rte_io_wmb();
 534                 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
 535                 qp->stats.dequeued_count += i;
 536         }
 537         return i;
 538 }
 539
 540 static void
 541 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 542 {
 543         uint32_t i;
 544
 545         for (i = 0 ; i < qp->entries_n; i++) {
 546                 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 547                         i * priv->wqe_set_size);
 548                 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
 549                                                                      (cseg + 1);
 550                 struct mlx5_wqe_umr_bsf_seg *bsf =
 551                         (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
 552                                                        priv->umr_wqe_size)) - 1;
 553                 struct mlx5_wqe_rseg *rseg;
 554
 555                 /* Init UMR WQE. */
 556                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
 557                                          (priv->umr_wqe_size / MLX5_WSEG_SIZE));
 558                 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 559                                        MLX5_COMP_MODE_OFFSET);
 560                 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
 561                 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
 562                 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
 563                 ucseg->ko_to_bs = rte_cpu_to_be_32
 564                         ((RTE_ALIGN(priv->max_segs_num, 4u) <<
 565                          MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
 566                 bsf->keytag = priv->keytag;
 567                 /* Init RDMA WRITE WQE. */
 568                 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 569                 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
 570                                       MLX5_COMP_MODE_OFFSET) |
 571                                       MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
 572                 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
 573                 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
 574         }
 575 }
 576
 577 static int
 578 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
 579                                   struct mlx5_crypto_qp *qp)
 580 {
 581         struct mlx5_umr_wqe *umr;
 582         uint32_t i;
 583         struct mlx5_devx_mkey_attr attr = {
 584                 .pd = priv->cdev->pdn,
 585                 .umr_en = 1,
 586                 .crypto_en = 1,
 587                 .set_remote_rw = 1,
 588                 .klm_num = RTE_ALIGN(priv->max_segs_num, 4),
 589         };
 590
 591         for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
 592            i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
 593                 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
 594                 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
 595                 if (!qp->mkey[i])
 596                         goto error;
 597         }
 598         return 0;
 599 error:
 600         DRV_LOG(ERR, "Failed to allocate indirect mkey.");
 601         mlx5_crypto_indirect_mkeys_release(qp, i);
 602         return -1;
 603 }
 604
 605 static int
 606 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 607                              const struct rte_cryptodev_qp_conf *qp_conf,
 608                              int socket_id)
 609 {
 610         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 611         struct mlx5_devx_qp_attr attr = {0};
 612         struct mlx5_crypto_qp *qp;
 613         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 614         uint32_t ret;
 615         uint32_t alloc_size = sizeof(*qp);
 616         struct mlx5_devx_cq_attr cq_attr = {
 617                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
 618         };
 619
 620         if (dev->data->queue_pairs[qp_id] != NULL)
 621                 mlx5_crypto_queue_pair_release(dev, qp_id);
 622         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 623         alloc_size += (sizeof(struct rte_crypto_op *) +
 624                        sizeof(struct mlx5_devx_obj *)) *
 625                        RTE_BIT32(log_nb_desc);
 626         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 627                                 socket_id);
 628         if (qp == NULL) {
 629                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 630                 rte_errno = ENOMEM;
 631                 return -rte_errno;
 632         }
 633         if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc,
 634                                 &cq_attr, socket_id) != 0) {
 635                 DRV_LOG(ERR, "Failed to create CQ.");
 636                 goto error;
 637         }
 638         attr.pd = priv->cdev->pdn;
 639         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 640         attr.cqn = qp->cq_obj.cq->id;
 641         attr.rq_size = 0;
 642         attr.sq_size = RTE_BIT32(log_nb_desc);
 643         attr.ts_format =
 644                 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 645         ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, log_nb_desc,
 646                                   &attr, socket_id);
 647         if (ret) {
 648                 DRV_LOG(ERR, "Failed to create QP.");
 649                 goto error;
 650         }
 651         if (mlx5_mr_ctrl_init(&qp->mr_ctrl, priv->cdev,
 652                               priv->dev_config.socket_id) != 0) {
 653                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 654                         (uint32_t)qp_id);
 655                 rte_errno = ENOMEM;
 656                 goto error;
 657         }
 658         /*
 659          * In Order to configure self loopback, when calling devx qp2rts the
 660          * remote QP id that is used is the id of the same QP.
 661          */
 662         if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id))
 663                 goto error;
 664         qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 665                                                            RTE_CACHE_LINE_SIZE);
 666         qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
 667         qp->entries_n = 1 << log_nb_desc;
 668         if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
 669                 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
 670                 rte_errno = ENOMEM;
 671                 goto error;
 672         }
 673         mlx5_crypto_qp_init(priv, qp);
 674         qp->priv = priv;
 675         dev->data->queue_pairs[qp_id] = qp;
 676         return 0;
 677 error:
 678         mlx5_crypto_qp_release(qp);
 679         return -1;
 680 }
 681
 682 static void
 683 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
 684                       struct rte_cryptodev_stats *stats)
 685 {
 686         int qp_id;
 687
 688         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 689                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 690
 691                 stats->enqueued_count += qp->stats.enqueued_count;
 692                 stats->dequeued_count += qp->stats.dequeued_count;
 693                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 694                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 695         }
 696 }
 697
 698 static void
 699 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
 700 {
 701         int qp_id;
 702
 703         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 704                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 705
 706                 memset(&qp->stats, 0, sizeof(qp->stats));
 707         }
 708 }
 709
 710 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 711         .dev_configure                  = mlx5_crypto_dev_configure,
 712         .dev_start                      = mlx5_crypto_dev_start,
 713         .dev_stop                       = mlx5_crypto_dev_stop,
 714         .dev_close                      = mlx5_crypto_dev_close,
 715         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 716         .stats_get                      = mlx5_crypto_stats_get,
 717         .stats_reset                    = mlx5_crypto_stats_reset,
 718         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 719         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 720         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 721         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 722         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 723         .sym_get_raw_dp_ctx_size        = NULL,
 724         .sym_configure_raw_dp_ctx       = NULL,
 725 };
 726
 727 static void
 728 mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv)
 729 {
 730         if (priv->uar != NULL) {
 731                 mlx5_glue->devx_free_uar(priv->uar);
 732                 priv->uar = NULL;
 733         }
 734 }
 735
 736 static int
 737 mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv)
 738 {
 739         priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
 740         if (priv->uar)
 741                 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
 742         if (priv->uar == NULL || priv->uar_addr == NULL) {
 743                 rte_errno = errno;
 744                 DRV_LOG(ERR, "Failed to allocate UAR.");
 745                 return -1;
 746         }
 747         return 0;
 748 }
 749
 750
 751 static int
 752 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 753 {
 754         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 755         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 756         unsigned long tmp;
 757         FILE *file;
 758         int ret;
 759         int i;
 760
 761         if (strcmp(key, "class") == 0)
 762                 return 0;
 763         if (strcmp(key, "wcs_file") == 0) {
 764                 file = fopen(val, "rb");
 765                 if (file == NULL) {
 766                         rte_errno = ENOTSUP;
 767                         return -rte_errno;
 768                 }
 769                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 770                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 771                         if (ret <= 0) {
 772                                 fclose(file);
 773                                 DRV_LOG(ERR,
 774                                         "Failed to read credential from file.");
 775                                 rte_errno = EINVAL;
 776                                 return -rte_errno;
 777                         }
 778                 }
 779                 fclose(file);
 780                 devarg_prms->login_devarg = true;
 781                 return 0;
 782         }
 783         errno = 0;
 784         tmp = strtoul(val, NULL, 0);
 785         if (errno) {
 786                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 787                 return -errno;
 788         }
 789         if (strcmp(key, "max_segs_num") == 0) {
 790                 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
 791                         DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
 792                                 " be less than %d.",
 793                                 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
 794                         rte_errno = EINVAL;
 795                         return -rte_errno;
 796                 }
 797                 devarg_prms->max_segs_num = (uint32_t)tmp;
 798         } else if (strcmp(key, "import_kek_id") == 0) {
 799                 attr->session_import_kek_ptr = (uint32_t)tmp;
 800         } else if (strcmp(key, "credential_id") == 0) {
 801                 attr->credential_pointer = (uint32_t)tmp;
 802         } else if (strcmp(key, "keytag") == 0) {
 803                 devarg_prms->keytag = tmp;
 804         } else {
 805                 DRV_LOG(WARNING, "Invalid key %s.", key);
 806         }
 807         return 0;
 808 }
 809
 810 static int
 811 mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 812                           struct mlx5_crypto_devarg_params *devarg_prms)
 813 {
 814         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 815         struct rte_kvargs *kvlist;
 816
 817         /* Default values. */
 818         attr->credential_pointer = 0;
 819         attr->session_import_kek_ptr = 0;
 820         devarg_prms->keytag = 0;
 821         devarg_prms->max_segs_num = 8;
 822         if (devargs == NULL) {
 823                 DRV_LOG(ERR,
 824         "No login devargs in order to enable crypto operations in the device.");
 825                 rte_errno = EINVAL;
 826                 return -1;
 827         }
 828         kvlist = rte_kvargs_parse(devargs->args, NULL);
 829         if (kvlist == NULL) {
 830                 DRV_LOG(ERR, "Failed to parse devargs.");
 831                 rte_errno = EINVAL;
 832                 return -1;
 833         }
 834         if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
 835                            devarg_prms) != 0) {
 836                 DRV_LOG(ERR, "Devargs handler function Failed.");
 837                 rte_kvargs_free(kvlist);
 838                 rte_errno = EINVAL;
 839                 return -1;
 840         }
 841         rte_kvargs_free(kvlist);
 842         if (devarg_prms->login_devarg == false) {
 843                 DRV_LOG(ERR,
 844         "No login credential devarg in order to enable crypto operations "
 845         "in the device.");
 846                 rte_errno = EINVAL;
 847                 return -1;
 848         }
 849         return 0;
 850 }
 851
 852 static int
 853 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 854 {
 855         struct rte_cryptodev *crypto_dev;
 856         struct mlx5_devx_obj *login;
 857         struct mlx5_crypto_priv *priv;
 858         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 859         struct rte_cryptodev_pmd_init_params init_params = {
 860                 .name = "",
 861                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 862                 .socket_id = cdev->dev->numa_node,
 863                 .max_nb_queue_pairs =
 864                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 865         };
 866         const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
 867         uint16_t rdmw_wqe_size;
 868         int ret;
 869
 870         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 871                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 872                 rte_errno = ENOTSUP;
 873                 return -rte_errno;
 874         }
 875         if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
 876                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 877                         "operations, maybe old FW/OFED version?");
 878                 rte_errno = ENOTSUP;
 879                 return -ENOTSUP;
 880         }
 881         ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms);
 882         if (ret) {
 883                 DRV_LOG(ERR, "Failed to parse devargs.");
 884                 return -rte_errno;
 885         }
 886         crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
 887                                               &init_params);
 888         if (crypto_dev == NULL) {
 889                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
 890                 return -ENODEV;
 891         }
 892         DRV_LOG(INFO,
 893                 "Crypto device %s was created successfully.", ibdev_name);
 894         crypto_dev->dev_ops = &mlx5_crypto_ops;
 895         crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 896         crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 897         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 898         crypto_dev->driver_id = mlx5_crypto_driver_id;
 899         priv = crypto_dev->data->dev_private;
 900         priv->cdev = cdev;
 901         priv->crypto_dev = crypto_dev;
 902         if (mlx5_crypto_uar_prepare(priv) != 0) {
 903                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 904                 return -1;
 905         }
 906         login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
 907                                                       &devarg_prms.login_attr);
 908         if (login == NULL) {
 909                 DRV_LOG(ERR, "Failed to configure login.");
 910                 mlx5_crypto_uar_release(priv);
 911                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 912                 return -rte_errno;
 913         }
 914         priv->login_obj = login;
 915         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 916         priv->max_segs_num = devarg_prms.max_segs_num;
 917         priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
 918                              sizeof(struct mlx5_wqe_cseg) +
 919                              sizeof(struct mlx5_wqe_umr_cseg) +
 920                              sizeof(struct mlx5_wqe_mkey_cseg) +
 921                              RTE_ALIGN(priv->max_segs_num, 4) *
 922                              sizeof(struct mlx5_wqe_dseg);
 923         rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
 924                               sizeof(struct mlx5_wqe_dseg) *
 925                               (priv->max_segs_num <= 2 ? 2 : 2 +
 926                                RTE_ALIGN(priv->max_segs_num - 2, 4));
 927         priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
 928         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
 929         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 930         pthread_mutex_lock(&priv_list_lock);
 931         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 932         pthread_mutex_unlock(&priv_list_lock);
 933
 934         rte_cryptodev_pmd_probing_finish(crypto_dev);
 935
 936         return 0;
 937 }
 938
 939 static int
 940 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
 941 {
 942         struct mlx5_crypto_priv *priv = NULL;
 943
 944         pthread_mutex_lock(&priv_list_lock);
 945         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 946                 if (priv->crypto_dev->device == cdev->dev)
 947                         break;
 948         if (priv)
 949                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 950         pthread_mutex_unlock(&priv_list_lock);
 951         if (priv) {
 952                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 953                 mlx5_crypto_uar_release(priv);
 954                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 955         }
 956         return 0;
 957 }
 958
 959 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 960                 {
 961                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 962                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 963                 },
 964                 {
 965                         .vendor_id = 0
 966                 }
 967 };
 968
 969 static struct mlx5_class_driver mlx5_crypto_driver = {
 970         .drv_class = MLX5_CLASS_CRYPTO,
 971         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
 972         .id_table = mlx5_crypto_pci_id_map,
 973         .probe = mlx5_crypto_dev_probe,
 974         .remove = mlx5_crypto_dev_remove,
 975 };
 976
 977 RTE_INIT(rte_mlx5_crypto_init)
 978 {
 979         pthread_mutex_init(&priv_list_lock, NULL);
 980         mlx5_common_init();
 981         if (mlx5_glue != NULL)
 982                 mlx5_class_driver_register(&mlx5_crypto_driver);
 983 }
 984
 985 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
 986                                mlx5_crypto_driver_id);
 987
 988 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
 989 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
 990 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
 991 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");