drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_errno.h>
   8 #include <rte_log.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_memory.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_common.h>
  14 #include <mlx5_devx_cmds.h>
  15 #include <mlx5_common_os.h>
  16
  17 #include "mlx5_crypto_utils.h"
  18 #include "mlx5_crypto.h"
  19
  20 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  21 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  22 #define MLX5_CRYPTO_MAX_QPS 1024
  23 #define MLX5_CRYPTO_MAX_SEGS 56
  24
  25 #define MLX5_CRYPTO_FEATURE_FLAGS \
  26         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  27          RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
  28          RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
  29          RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
  30          RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
  31          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  32          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  33
  34 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  35                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  36 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
  37
  38 int mlx5_crypto_logtype;
  39
  40 uint8_t mlx5_crypto_driver_id;
  41
  42 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  43         {               /* AES XTS */
  44                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  45                 {.sym = {
  46                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  47                         {.cipher = {
  48                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  49                                 .block_size = 16,
  50                                 .key_size = {
  51                                         .min = 32,
  52                                         .max = 64,
  53                                         .increment = 32
  54                                 },
  55                                 .iv_size = {
  56                                         .min = 16,
  57                                         .max = 16,
  58                                         .increment = 0
  59                                 },
  60                                 .dataunit_set =
  61                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  62                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
  63                         }, }
  64                 }, }
  65         },
  66 };
  67
  68 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  69
  70 static const struct rte_driver mlx5_drv = {
  71         .name = mlx5_crypto_drv_name,
  72         .alias = mlx5_crypto_drv_name
  73 };
  74
  75 static struct cryptodev_driver mlx5_cryptodev_driver;
  76
  77 struct mlx5_crypto_session {
  78         uint32_t bs_bpt_eo_es;
  79         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  80          * saved in big endian format.
  81          */
  82         uint32_t bsp_res;
  83         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  84          * endian format.
  85          */
  86         uint32_t iv_offset:16;
  87         /**< Starting point for Initialisation Vector. */
  88         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  89         uint32_t dek_id; /**< DEK ID */
  90 } __rte_packed;
  91
  92 static void
  93 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  94                           struct rte_cryptodev_info *dev_info)
  95 {
  96         RTE_SET_USED(dev);
  97         if (dev_info != NULL) {
  98                 dev_info->driver_id = mlx5_crypto_driver_id;
  99                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 100                 dev_info->capabilities = mlx5_crypto_caps;
 101                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 102                 dev_info->min_mbuf_headroom_req = 0;
 103                 dev_info->min_mbuf_tailroom_req = 0;
 104                 dev_info->sym.max_nb_sessions = 0;
 105                 /*
 106                  * If 0, the device does not have any limitation in number of
 107                  * sessions that can be used.
 108                  */
 109         }
 110 }
 111
 112 static int
 113 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 114                           struct rte_cryptodev_config *config)
 115 {
 116         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 117
 118         if (config == NULL) {
 119                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 120                 return -EINVAL;
 121         }
 122         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 123                 DRV_LOG(ERR,
 124                         "Disabled symmetric crypto feature is not supported.");
 125                 return -ENOTSUP;
 126         }
 127         if (mlx5_crypto_dek_setup(priv) != 0) {
 128                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 129                 return -ENOMEM;
 130         }
 131         priv->dev_config = *config;
 132         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 133         return 0;
 134 }
 135
 136 static void
 137 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 138 {
 139         RTE_SET_USED(dev);
 140 }
 141
 142 static int
 143 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 144 {
 145         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 146
 147         return mlx5_dev_mempool_subscribe(priv->cdev);
 148 }
 149
 150 static int
 151 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 152 {
 153         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 154
 155         mlx5_crypto_dek_unset(priv);
 156         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 157         return 0;
 158 }
 159
 160 static unsigned int
 161 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 162 {
 163         return sizeof(struct mlx5_crypto_session);
 164 }
 165
 166 static int
 167 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 168                                   struct rte_crypto_sym_xform *xform,
 169                                   struct rte_cryptodev_sym_session *session,
 170                                   struct rte_mempool *mp)
 171 {
 172         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 173         struct mlx5_crypto_session *sess_private_data;
 174         struct rte_crypto_cipher_xform *cipher;
 175         uint8_t encryption_order;
 176         int ret;
 177
 178         if (unlikely(xform->next != NULL)) {
 179                 DRV_LOG(ERR, "Xform next is not supported.");
 180                 return -ENOTSUP;
 181         }
 182         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 183                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 184                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 185                 return -ENOTSUP;
 186         }
 187         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 188         if (ret != 0) {
 189                 DRV_LOG(ERR,
 190                         "Failed to get session %p private data from mempool.",
 191                         sess_private_data);
 192                 return -ENOMEM;
 193         }
 194         cipher = &xform->cipher;
 195         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 196         if (sess_private_data->dek == NULL) {
 197                 rte_mempool_put(mp, sess_private_data);
 198                 DRV_LOG(ERR, "Failed to prepare dek.");
 199                 return -ENOMEM;
 200         }
 201         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 202                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 203         else
 204                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 205         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 206                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 207                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 208                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 209                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 210         switch (xform->cipher.dataunit_len) {
 211         case 0:
 212                 sess_private_data->bsp_res = 0;
 213                 break;
 214         case 512:
 215                 sess_private_data->bsp_res = rte_cpu_to_be_32
 216                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 217                                              MLX5_BLOCK_SIZE_OFFSET);
 218                 break;
 219         case 4096:
 220                 sess_private_data->bsp_res = rte_cpu_to_be_32
 221                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 222                                              MLX5_BLOCK_SIZE_OFFSET);
 223                 break;
 224         default:
 225                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 226                 return -ENOTSUP;
 227         }
 228         sess_private_data->iv_offset = cipher->iv.offset;
 229         sess_private_data->dek_id =
 230                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 231                                          0xffffff);
 232         set_sym_session_private_data(session, dev->driver_id,
 233                                      sess_private_data);
 234         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 235         return 0;
 236 }
 237
 238 static void
 239 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 240                               struct rte_cryptodev_sym_session *sess)
 241 {
 242         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 243         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 244                                                                 dev->driver_id);
 245
 246         if (unlikely(spriv == NULL)) {
 247                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 248                 return;
 249         }
 250         mlx5_crypto_dek_destroy(priv, spriv->dek);
 251         set_sym_session_private_data(sess, dev->driver_id, NULL);
 252         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 253         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 254 }
 255
 256 static void
 257 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n)
 258 {
 259         uint16_t i;
 260
 261         for (i = 0; i < n; i++)
 262                 if (qp->mkey[i])
 263                         claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
 264 }
 265
 266 static void
 267 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp)
 268 {
 269         if (qp == NULL)
 270                 return;
 271         mlx5_devx_qp_destroy(&qp->qp_obj);
 272         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 273         mlx5_devx_cq_destroy(&qp->cq_obj);
 274         rte_free(qp);
 275 }
 276
 277 static int
 278 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 279 {
 280         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 281
 282         mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
 283         mlx5_crypto_qp_release(qp);
 284         dev->data->queue_pairs[qp_id] = NULL;
 285         return 0;
 286 }
 287
 288 static __rte_noinline uint32_t
 289 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 290 {
 291         uint32_t bl = op->sym->cipher.data.length;
 292
 293         switch (bl) {
 294         case (1 << 20):
 295                 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
 296         case (1 << 12):
 297                 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
 298                                 MLX5_BLOCK_SIZE_OFFSET);
 299         case (1 << 9):
 300                 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
 301         default:
 302                 DRV_LOG(ERR, "Unknown block size: %u.", bl);
 303                 return UINT32_MAX;
 304         }
 305 }
 306
 307 static __rte_always_inline uint32_t
 308 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 309                       struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
 310                       uint32_t offset, uint32_t *remain)
 311 {
 312         uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 313         uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 314
 315         if (data_len > *remain)
 316                 data_len = *remain;
 317         *remain -= data_len;
 318         klm->bcount = rte_cpu_to_be_32(data_len);
 319         klm->pbuf = rte_cpu_to_be_64(addr);
 320         klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf);
 321         return klm->lkey;
 322
 323 }
 324
 325 static __rte_always_inline uint32_t
 326 mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 327                      struct rte_crypto_op *op, struct rte_mbuf *mbuf,
 328                      struct mlx5_wqe_dseg *klm)
 329 {
 330         uint32_t remain_len = op->sym->cipher.data.length;
 331         uint32_t nb_segs = mbuf->nb_segs;
 332         uint32_t klm_n = 1u;
 333
 334         /* First mbuf needs to take the cipher offset. */
 335         if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
 336                      op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 337                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 338                 return 0;
 339         }
 340         while (remain_len) {
 341                 nb_segs--;
 342                 mbuf = mbuf->next;
 343                 if (unlikely(mbuf == NULL || nb_segs == 0)) {
 344                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 345                         return 0;
 346                 }
 347                 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
 348                                                  &remain_len) == UINT32_MAX)) {
 349                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 350                         return 0;
 351                 }
 352                 klm_n++;
 353         }
 354         return klm_n;
 355 }
 356
 357 static __rte_always_inline int
 358 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 359                          struct mlx5_crypto_qp *qp,
 360                          struct rte_crypto_op *op,
 361                          struct mlx5_umr_wqe *umr)
 362 {
 363         struct mlx5_crypto_session *sess = get_sym_session_private_data
 364                                 (op->sym->session, mlx5_crypto_driver_id);
 365         struct mlx5_wqe_cseg *cseg = &umr->ctr;
 366         struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
 367         struct mlx5_wqe_dseg *klms = &umr->kseg[0];
 368         struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
 369                                       RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
 370         uint32_t ds;
 371         bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 372         /* Set UMR WQE. */
 373         uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
 374                                    ipl ? op->sym->m_src : op->sym->m_dst, klms);
 375
 376         if (unlikely(klm_n == 0))
 377                 return 0;
 378         bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
 379         if (unlikely(!sess->bsp_res)) {
 380                 bsf->bsp_res = mlx5_crypto_get_block_size(op);
 381                 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
 382                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 383                         return 0;
 384                 }
 385         } else {
 386                 bsf->bsp_res = sess->bsp_res;
 387         }
 388         bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
 389         memcpy(bsf->xts_initial_tweak,
 390                rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
 391         bsf->res_dp = sess->dek_id;
 392         mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
 393         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
 394         qp->db_pi += priv->umr_wqe_stride;
 395         /* Set RDMA_WRITE WQE. */
 396         cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 397         klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 398         if (!ipl) {
 399                 klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
 400                                              klms);
 401                 if (unlikely(klm_n == 0))
 402                         return 0;
 403         } else {
 404                 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
 405         }
 406         ds = 2 + klm_n;
 407         cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 408         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 409                                                         MLX5_OPCODE_RDMA_WRITE);
 410         ds = RTE_ALIGN(ds, 4);
 411         qp->db_pi += ds >> 2;
 412         /* Set NOP WQE if needed. */
 413         if (priv->max_rdmar_ds > ds) {
 414                 cseg += ds;
 415                 ds = priv->max_rdmar_ds - ds;
 416                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 417                 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 418                                                                MLX5_OPCODE_NOP);
 419                 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
 420         }
 421         qp->wqe = (uint8_t *)cseg;
 422         return 1;
 423 }
 424
 425 static __rte_always_inline void
 426 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
 427 {
 428 #ifdef RTE_ARCH_64
 429         *priv->uar_addr = val;
 430 #else /* !RTE_ARCH_64 */
 431         rte_spinlock_lock(&priv->uar32_sl);
 432         *(volatile uint32_t *)priv->uar_addr = val;
 433         rte_io_wmb();
 434         *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
 435         rte_spinlock_unlock(&priv->uar32_sl);
 436 #endif
 437 }
 438
 439 static uint16_t
 440 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 441                           uint16_t nb_ops)
 442 {
 443         struct mlx5_crypto_qp *qp = queue_pair;
 444         struct mlx5_crypto_priv *priv = qp->priv;
 445         struct mlx5_umr_wqe *umr;
 446         struct rte_crypto_op *op;
 447         uint16_t mask = qp->entries_n - 1;
 448         uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
 449         uint32_t idx;
 450
 451         if (remain < nb_ops)
 452                 nb_ops = remain;
 453         else
 454                 remain = nb_ops;
 455         if (unlikely(remain == 0))
 456                 return 0;
 457         do {
 458                 idx = qp->pi & mask;
 459                 op = *ops++;
 460                 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 461                         priv->wqe_set_size * idx);
 462                 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
 463                         qp->stats.enqueue_err_count++;
 464                         if (remain != nb_ops) {
 465                                 qp->stats.enqueued_count -= remain;
 466                                 break;
 467                         }
 468                         return 0;
 469                 }
 470                 qp->ops[idx] = op;
 471                 qp->pi++;
 472         } while (--remain);
 473         qp->stats.enqueued_count += nb_ops;
 474         rte_io_wmb();
 475         qp->qp_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 476         rte_wmb();
 477         mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
 478         rte_wmb();
 479         return nb_ops;
 480 }
 481
 482 static __rte_noinline void
 483 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 484 {
 485         const uint32_t idx = qp->ci & (qp->entries_n - 1);
 486         volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
 487                                                         &qp->cq_obj.cqes[idx];
 488
 489         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 490         qp->stats.dequeue_err_count++;
 491         DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 492 }
 493
 494 static uint16_t
 495 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 496                           uint16_t nb_ops)
 497 {
 498         struct mlx5_crypto_qp *qp = queue_pair;
 499         volatile struct mlx5_cqe *restrict cqe;
 500         struct rte_crypto_op *restrict op;
 501         const unsigned int cq_size = qp->entries_n;
 502         const unsigned int mask = cq_size - 1;
 503         uint32_t idx;
 504         uint32_t next_idx = qp->ci & mask;
 505         const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
 506         uint16_t i = 0;
 507         int ret;
 508
 509         if (unlikely(max == 0))
 510                 return 0;
 511         do {
 512                 idx = next_idx;
 513                 next_idx = (qp->ci + 1) & mask;
 514                 op = qp->ops[idx];
 515                 cqe = &qp->cq_obj.cqes[idx];
 516                 ret = check_cqe(cqe, cq_size, qp->ci);
 517                 rte_io_rmb();
 518                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 519                         if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
 520                                 mlx5_crypto_cqe_err_handle(qp, op);
 521                         break;
 522                 }
 523                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 524                 ops[i++] = op;
 525                 qp->ci++;
 526         } while (i < max);
 527         if (likely(i != 0)) {
 528                 rte_io_wmb();
 529                 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
 530                 qp->stats.dequeued_count += i;
 531         }
 532         return i;
 533 }
 534
 535 static void
 536 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 537 {
 538         uint32_t i;
 539
 540         for (i = 0 ; i < qp->entries_n; i++) {
 541                 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 542                         i * priv->wqe_set_size);
 543                 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
 544                                                                      (cseg + 1);
 545                 struct mlx5_wqe_umr_bsf_seg *bsf =
 546                         (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
 547                                                        priv->umr_wqe_size)) - 1;
 548                 struct mlx5_wqe_rseg *rseg;
 549
 550                 /* Init UMR WQE. */
 551                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
 552                                          (priv->umr_wqe_size / MLX5_WSEG_SIZE));
 553                 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 554                                        MLX5_COMP_MODE_OFFSET);
 555                 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
 556                 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
 557                 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
 558                 ucseg->ko_to_bs = rte_cpu_to_be_32
 559                         ((RTE_ALIGN(priv->max_segs_num, 4u) <<
 560                          MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
 561                 bsf->keytag = priv->keytag;
 562                 /* Init RDMA WRITE WQE. */
 563                 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 564                 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
 565                                       MLX5_COMP_MODE_OFFSET) |
 566                                       MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
 567                 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
 568                 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
 569         }
 570 }
 571
 572 static int
 573 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
 574                                   struct mlx5_crypto_qp *qp)
 575 {
 576         struct mlx5_umr_wqe *umr;
 577         uint32_t i;
 578         struct mlx5_devx_mkey_attr attr = {
 579                 .pd = priv->cdev->pdn,
 580                 .umr_en = 1,
 581                 .crypto_en = 1,
 582                 .set_remote_rw = 1,
 583                 .klm_num = RTE_ALIGN(priv->max_segs_num, 4),
 584         };
 585
 586         for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
 587            i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
 588                 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
 589                 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
 590                 if (!qp->mkey[i])
 591                         goto error;
 592         }
 593         return 0;
 594 error:
 595         DRV_LOG(ERR, "Failed to allocate indirect mkey.");
 596         mlx5_crypto_indirect_mkeys_release(qp, i);
 597         return -1;
 598 }
 599
 600 static int
 601 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 602                              const struct rte_cryptodev_qp_conf *qp_conf,
 603                              int socket_id)
 604 {
 605         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 606         struct mlx5_devx_qp_attr attr = {0};
 607         struct mlx5_crypto_qp *qp;
 608         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 609         uint32_t ret;
 610         uint32_t alloc_size = sizeof(*qp);
 611         struct mlx5_devx_cq_attr cq_attr = {
 612                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
 613         };
 614
 615         if (dev->data->queue_pairs[qp_id] != NULL)
 616                 mlx5_crypto_queue_pair_release(dev, qp_id);
 617         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 618         alloc_size += (sizeof(struct rte_crypto_op *) +
 619                        sizeof(struct mlx5_devx_obj *)) *
 620                        RTE_BIT32(log_nb_desc);
 621         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 622                                 socket_id);
 623         if (qp == NULL) {
 624                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 625                 rte_errno = ENOMEM;
 626                 return -rte_errno;
 627         }
 628         if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc,
 629                                 &cq_attr, socket_id) != 0) {
 630                 DRV_LOG(ERR, "Failed to create CQ.");
 631                 goto error;
 632         }
 633         attr.pd = priv->cdev->pdn;
 634         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 635         attr.cqn = qp->cq_obj.cq->id;
 636         attr.rq_size = 0;
 637         attr.sq_size = RTE_BIT32(log_nb_desc);
 638         attr.ts_format =
 639                 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 640         ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, log_nb_desc,
 641                                   &attr, socket_id);
 642         if (ret) {
 643                 DRV_LOG(ERR, "Failed to create QP.");
 644                 goto error;
 645         }
 646         if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
 647                               priv->dev_config.socket_id) != 0) {
 648                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 649                         (uint32_t)qp_id);
 650                 rte_errno = ENOMEM;
 651                 goto error;
 652         }
 653         /*
 654          * In Order to configure self loopback, when calling devx qp2rts the
 655          * remote QP id that is used is the id of the same QP.
 656          */
 657         if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id))
 658                 goto error;
 659         qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 660                                                            RTE_CACHE_LINE_SIZE);
 661         qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
 662         qp->entries_n = 1 << log_nb_desc;
 663         if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
 664                 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
 665                 rte_errno = ENOMEM;
 666                 goto error;
 667         }
 668         mlx5_crypto_qp_init(priv, qp);
 669         qp->priv = priv;
 670         dev->data->queue_pairs[qp_id] = qp;
 671         return 0;
 672 error:
 673         mlx5_crypto_qp_release(qp);
 674         return -1;
 675 }
 676
 677 static void
 678 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
 679                       struct rte_cryptodev_stats *stats)
 680 {
 681         int qp_id;
 682
 683         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 684                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 685
 686                 stats->enqueued_count += qp->stats.enqueued_count;
 687                 stats->dequeued_count += qp->stats.dequeued_count;
 688                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 689                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 690         }
 691 }
 692
 693 static void
 694 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
 695 {
 696         int qp_id;
 697
 698         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 699                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 700
 701                 memset(&qp->stats, 0, sizeof(qp->stats));
 702         }
 703 }
 704
 705 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 706         .dev_configure                  = mlx5_crypto_dev_configure,
 707         .dev_start                      = mlx5_crypto_dev_start,
 708         .dev_stop                       = mlx5_crypto_dev_stop,
 709         .dev_close                      = mlx5_crypto_dev_close,
 710         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 711         .stats_get                      = mlx5_crypto_stats_get,
 712         .stats_reset                    = mlx5_crypto_stats_reset,
 713         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 714         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 715         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 716         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 717         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 718         .sym_get_raw_dp_ctx_size        = NULL,
 719         .sym_configure_raw_dp_ctx       = NULL,
 720 };
 721
 722 static void
 723 mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv)
 724 {
 725         if (priv->uar != NULL) {
 726                 mlx5_glue->devx_free_uar(priv->uar);
 727                 priv->uar = NULL;
 728         }
 729 }
 730
 731 static int
 732 mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv)
 733 {
 734         priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
 735         if (priv->uar)
 736                 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
 737         if (priv->uar == NULL || priv->uar_addr == NULL) {
 738                 rte_errno = errno;
 739                 DRV_LOG(ERR, "Failed to allocate UAR.");
 740                 return -1;
 741         }
 742         return 0;
 743 }
 744
 745
 746 static int
 747 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 748 {
 749         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 750         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 751         unsigned long tmp;
 752         FILE *file;
 753         int ret;
 754         int i;
 755
 756         if (strcmp(key, "class") == 0)
 757                 return 0;
 758         if (strcmp(key, "wcs_file") == 0) {
 759                 file = fopen(val, "rb");
 760                 if (file == NULL) {
 761                         rte_errno = ENOTSUP;
 762                         return -rte_errno;
 763                 }
 764                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 765                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 766                         if (ret <= 0) {
 767                                 fclose(file);
 768                                 DRV_LOG(ERR,
 769                                         "Failed to read credential from file.");
 770                                 rte_errno = EINVAL;
 771                                 return -rte_errno;
 772                         }
 773                 }
 774                 fclose(file);
 775                 devarg_prms->login_devarg = true;
 776                 return 0;
 777         }
 778         errno = 0;
 779         tmp = strtoul(val, NULL, 0);
 780         if (errno) {
 781                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 782                 return -errno;
 783         }
 784         if (strcmp(key, "max_segs_num") == 0) {
 785                 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
 786                         DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
 787                                 " be less than %d.",
 788                                 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
 789                         rte_errno = EINVAL;
 790                         return -rte_errno;
 791                 }
 792                 devarg_prms->max_segs_num = (uint32_t)tmp;
 793         } else if (strcmp(key, "import_kek_id") == 0) {
 794                 attr->session_import_kek_ptr = (uint32_t)tmp;
 795         } else if (strcmp(key, "credential_id") == 0) {
 796                 attr->credential_pointer = (uint32_t)tmp;
 797         } else if (strcmp(key, "keytag") == 0) {
 798                 devarg_prms->keytag = tmp;
 799         } else {
 800                 DRV_LOG(WARNING, "Invalid key %s.", key);
 801         }
 802         return 0;
 803 }
 804
 805 static int
 806 mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 807                           struct mlx5_crypto_devarg_params *devarg_prms)
 808 {
 809         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 810         struct rte_kvargs *kvlist;
 811
 812         /* Default values. */
 813         attr->credential_pointer = 0;
 814         attr->session_import_kek_ptr = 0;
 815         devarg_prms->keytag = 0;
 816         devarg_prms->max_segs_num = 8;
 817         if (devargs == NULL) {
 818                 DRV_LOG(ERR,
 819         "No login devargs in order to enable crypto operations in the device.");
 820                 rte_errno = EINVAL;
 821                 return -1;
 822         }
 823         kvlist = rte_kvargs_parse(devargs->args, NULL);
 824         if (kvlist == NULL) {
 825                 DRV_LOG(ERR, "Failed to parse devargs.");
 826                 rte_errno = EINVAL;
 827                 return -1;
 828         }
 829         if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
 830                            devarg_prms) != 0) {
 831                 DRV_LOG(ERR, "Devargs handler function Failed.");
 832                 rte_kvargs_free(kvlist);
 833                 rte_errno = EINVAL;
 834                 return -1;
 835         }
 836         rte_kvargs_free(kvlist);
 837         if (devarg_prms->login_devarg == false) {
 838                 DRV_LOG(ERR,
 839         "No login credential devarg in order to enable crypto operations "
 840         "in the device.");
 841                 rte_errno = EINVAL;
 842                 return -1;
 843         }
 844         return 0;
 845 }
 846
 847 static int
 848 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 849 {
 850         struct rte_cryptodev *crypto_dev;
 851         struct mlx5_devx_obj *login;
 852         struct mlx5_crypto_priv *priv;
 853         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 854         struct rte_cryptodev_pmd_init_params init_params = {
 855                 .name = "",
 856                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 857                 .socket_id = cdev->dev->numa_node,
 858                 .max_nb_queue_pairs =
 859                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 860         };
 861         const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
 862         uint16_t rdmw_wqe_size;
 863         int ret;
 864
 865         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 866                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 867                 rte_errno = ENOTSUP;
 868                 return -rte_errno;
 869         }
 870         if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
 871                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 872                         "operations, maybe old FW/OFED version?");
 873                 rte_errno = ENOTSUP;
 874                 return -ENOTSUP;
 875         }
 876         ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms);
 877         if (ret) {
 878                 DRV_LOG(ERR, "Failed to parse devargs.");
 879                 return -rte_errno;
 880         }
 881         login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
 882                                                       &devarg_prms.login_attr);
 883         if (login == NULL) {
 884                 DRV_LOG(ERR, "Failed to configure login.");
 885                 return -rte_errno;
 886         }
 887         crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
 888                                               &init_params);
 889         if (crypto_dev == NULL) {
 890                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
 891                 return -ENODEV;
 892         }
 893         DRV_LOG(INFO,
 894                 "Crypto device %s was created successfully.", ibdev_name);
 895         crypto_dev->dev_ops = &mlx5_crypto_ops;
 896         crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 897         crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 898         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 899         crypto_dev->driver_id = mlx5_crypto_driver_id;
 900         priv = crypto_dev->data->dev_private;
 901         priv->cdev = cdev;
 902         priv->login_obj = login;
 903         priv->crypto_dev = crypto_dev;
 904         if (mlx5_crypto_uar_prepare(priv) != 0) {
 905                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 906                 return -1;
 907         }
 908         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 909         priv->max_segs_num = devarg_prms.max_segs_num;
 910         priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
 911                              sizeof(struct mlx5_umr_wqe) +
 912                              RTE_ALIGN(priv->max_segs_num, 4) *
 913                              sizeof(struct mlx5_wqe_dseg);
 914         rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
 915                               sizeof(struct mlx5_wqe_dseg) *
 916                               (priv->max_segs_num <= 2 ? 2 : 2 +
 917                                RTE_ALIGN(priv->max_segs_num - 2, 4));
 918         priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
 919         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
 920         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 921         pthread_mutex_lock(&priv_list_lock);
 922         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 923         pthread_mutex_unlock(&priv_list_lock);
 924
 925         rte_cryptodev_pmd_probing_finish(crypto_dev);
 926
 927         return 0;
 928 }
 929
 930 static int
 931 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
 932 {
 933         struct mlx5_crypto_priv *priv = NULL;
 934
 935         pthread_mutex_lock(&priv_list_lock);
 936         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 937                 if (priv->crypto_dev->device == cdev->dev)
 938                         break;
 939         if (priv)
 940                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 941         pthread_mutex_unlock(&priv_list_lock);
 942         if (priv) {
 943                 mlx5_crypto_uar_release(priv);
 944                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 945                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 946         }
 947         return 0;
 948 }
 949
 950 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 951                 {
 952                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 953                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 954                 },
 955                 {
 956                         .vendor_id = 0
 957                 }
 958 };
 959
 960 static struct mlx5_class_driver mlx5_crypto_driver = {
 961         .drv_class = MLX5_CLASS_CRYPTO,
 962         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
 963         .id_table = mlx5_crypto_pci_id_map,
 964         .probe = mlx5_crypto_dev_probe,
 965         .remove = mlx5_crypto_dev_remove,
 966 };
 967
 968 RTE_INIT(rte_mlx5_crypto_init)
 969 {
 970         mlx5_common_init();
 971         if (mlx5_glue != NULL)
 972                 mlx5_class_driver_register(&mlx5_crypto_driver);
 973 }
 974
 975 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
 976                                mlx5_crypto_driver_id);
 977
 978 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
 979 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
 980 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
 981 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");