drivers/crypto/mlx5/mlx5_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mempool.h>
   7 #include <rte_eal_paging.h>
   8 #include <rte_errno.h>
   9 #include <rte_log.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_memory.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_common.h>
  15 #include <mlx5_devx_cmds.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_crypto_utils.h"
  19 #include "mlx5_crypto.h"
  20
  21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
  22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
  23 #define MLX5_CRYPTO_MAX_QPS 1024
  24 #define MLX5_CRYPTO_MAX_SEGS 56
  25
  26 #define MLX5_CRYPTO_FEATURE_FLAGS \
  27         (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
  28          RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
  29          RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
  30          RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
  31          RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
  32          RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
  33          RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
  34
  35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
  36                                 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
  37 static pthread_mutex_t priv_list_lock;
  38
  39 int mlx5_crypto_logtype;
  40
  41 uint8_t mlx5_crypto_driver_id;
  42
  43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
  44         {               /* AES XTS */
  45                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  46                 {.sym = {
  47                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  48                         {.cipher = {
  49                                 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
  50                                 .block_size = 16,
  51                                 .key_size = {
  52                                         .min = 32,
  53                                         .max = 64,
  54                                         .increment = 32
  55                                 },
  56                                 .iv_size = {
  57                                         .min = 16,
  58                                         .max = 16,
  59                                         .increment = 0
  60                                 },
  61                                 .dataunit_set =
  62                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
  63                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
  64                         }, }
  65                 }, }
  66         },
  67 };
  68
  69 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
  70
  71 static const struct rte_driver mlx5_drv = {
  72         .name = mlx5_crypto_drv_name,
  73         .alias = mlx5_crypto_drv_name
  74 };
  75
  76 static struct cryptodev_driver mlx5_cryptodev_driver;
  77
  78 struct mlx5_crypto_session {
  79         uint32_t bs_bpt_eo_es;
  80         /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
  81          * saved in big endian format.
  82          */
  83         uint32_t bsp_res;
  84         /**< crypto_block_size_pointer and reserved 24 bits saved in big
  85          * endian format.
  86          */
  87         uint32_t iv_offset:16;
  88         /**< Starting point for Initialisation Vector. */
  89         struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
  90         uint32_t dek_id; /**< DEK ID */
  91 } __rte_packed;
  92
  93 static void
  94 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
  95                           struct rte_cryptodev_info *dev_info)
  96 {
  97         RTE_SET_USED(dev);
  98         if (dev_info != NULL) {
  99                 dev_info->driver_id = mlx5_crypto_driver_id;
 100                 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 101                 dev_info->capabilities = mlx5_crypto_caps;
 102                 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 103                 dev_info->min_mbuf_headroom_req = 0;
 104                 dev_info->min_mbuf_tailroom_req = 0;
 105                 dev_info->sym.max_nb_sessions = 0;
 106                 /*
 107                  * If 0, the device does not have any limitation in number of
 108                  * sessions that can be used.
 109                  */
 110         }
 111 }
 112
 113 static int
 114 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 115                           struct rte_cryptodev_config *config)
 116 {
 117         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 118
 119         if (config == NULL) {
 120                 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
 121                 return -EINVAL;
 122         }
 123         if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
 124                 DRV_LOG(ERR,
 125                         "Disabled symmetric crypto feature is not supported.");
 126                 return -ENOTSUP;
 127         }
 128         if (mlx5_crypto_dek_setup(priv) != 0) {
 129                 DRV_LOG(ERR, "Dek hash list creation has failed.");
 130                 return -ENOMEM;
 131         }
 132         priv->dev_config = *config;
 133         DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 134         return 0;
 135 }
 136
 137 static void
 138 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 139 {
 140         RTE_SET_USED(dev);
 141 }
 142
 143 static int
 144 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 145 {
 146         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 147
 148         return mlx5_dev_mempool_subscribe(priv->cdev);
 149 }
 150
 151 static int
 152 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 153 {
 154         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 155
 156         mlx5_crypto_dek_unset(priv);
 157         DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 158         return 0;
 159 }
 160
 161 static unsigned int
 162 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 163 {
 164         return sizeof(struct mlx5_crypto_session);
 165 }
 166
 167 static int
 168 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 169                                   struct rte_crypto_sym_xform *xform,
 170                                   struct rte_cryptodev_sym_session *session,
 171                                   struct rte_mempool *mp)
 172 {
 173         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 174         struct mlx5_crypto_session *sess_private_data;
 175         struct rte_crypto_cipher_xform *cipher;
 176         uint8_t encryption_order;
 177         int ret;
 178
 179         if (unlikely(xform->next != NULL)) {
 180                 DRV_LOG(ERR, "Xform next is not supported.");
 181                 return -ENOTSUP;
 182         }
 183         if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
 184                      (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
 185                 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
 186                 return -ENOTSUP;
 187         }
 188         ret = rte_mempool_get(mp, (void *)&sess_private_data);
 189         if (ret != 0) {
 190                 DRV_LOG(ERR,
 191                         "Failed to get session %p private data from mempool.",
 192                         sess_private_data);
 193                 return -ENOMEM;
 194         }
 195         cipher = &xform->cipher;
 196         sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
 197         if (sess_private_data->dek == NULL) {
 198                 rte_mempool_put(mp, sess_private_data);
 199                 DRV_LOG(ERR, "Failed to prepare dek.");
 200                 return -ENOMEM;
 201         }
 202         if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 203                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
 204         else
 205                 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
 206         sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
 207                         (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
 208                          MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 209                          encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 210                          MLX5_ENCRYPTION_STANDARD_AES_XTS);
 211         switch (xform->cipher.dataunit_len) {
 212         case 0:
 213                 sess_private_data->bsp_res = 0;
 214                 break;
 215         case 512:
 216                 sess_private_data->bsp_res = rte_cpu_to_be_32
 217                                              ((uint32_t)MLX5_BLOCK_SIZE_512B <<
 218                                              MLX5_BLOCK_SIZE_OFFSET);
 219                 break;
 220         case 4096:
 221                 sess_private_data->bsp_res = rte_cpu_to_be_32
 222                                              ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
 223                                              MLX5_BLOCK_SIZE_OFFSET);
 224                 break;
 225         default:
 226                 DRV_LOG(ERR, "Cipher data unit length is not supported.");
 227                 return -ENOTSUP;
 228         }
 229         sess_private_data->iv_offset = cipher->iv.offset;
 230         sess_private_data->dek_id =
 231                         rte_cpu_to_be_32(sess_private_data->dek->obj->id &
 232                                          0xffffff);
 233         set_sym_session_private_data(session, dev->driver_id,
 234                                      sess_private_data);
 235         DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
 236         return 0;
 237 }
 238
 239 static void
 240 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 241                               struct rte_cryptodev_sym_session *sess)
 242 {
 243         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 244         struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
 245                                                                 dev->driver_id);
 246
 247         if (unlikely(spriv == NULL)) {
 248                 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
 249                 return;
 250         }
 251         mlx5_crypto_dek_destroy(priv, spriv->dek);
 252         set_sym_session_private_data(sess, dev->driver_id, NULL);
 253         rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
 254         DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 255 }
 256
 257 static void
 258 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n)
 259 {
 260         uint16_t i;
 261
 262         for (i = 0; i < n; i++)
 263                 if (qp->mkey[i])
 264                         claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
 265 }
 266
 267 static void
 268 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp)
 269 {
 270         if (qp == NULL)
 271                 return;
 272         mlx5_devx_qp_destroy(&qp->qp_obj);
 273         mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 274         mlx5_devx_cq_destroy(&qp->cq_obj);
 275         rte_free(qp);
 276 }
 277
 278 static int
 279 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 280 {
 281         struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 282
 283         mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
 284         mlx5_crypto_qp_release(qp);
 285         dev->data->queue_pairs[qp_id] = NULL;
 286         return 0;
 287 }
 288
 289 static __rte_noinline uint32_t
 290 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 291 {
 292         uint32_t bl = op->sym->cipher.data.length;
 293
 294         switch (bl) {
 295         case (1 << 20):
 296                 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
 297         case (1 << 12):
 298                 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
 299                                 MLX5_BLOCK_SIZE_OFFSET);
 300         case (1 << 9):
 301                 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
 302         default:
 303                 DRV_LOG(ERR, "Unknown block size: %u.", bl);
 304                 return UINT32_MAX;
 305         }
 306 }
 307
 308 static __rte_always_inline uint32_t
 309 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 310                       struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
 311                       uint32_t offset, uint32_t *remain)
 312 {
 313         uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 314         uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 315
 316         if (data_len > *remain)
 317                 data_len = *remain;
 318         *remain -= data_len;
 319         klm->bcount = rte_cpu_to_be_32(data_len);
 320         klm->pbuf = rte_cpu_to_be_64(addr);
 321         klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf);
 322         return klm->lkey;
 323
 324 }
 325
 326 static __rte_always_inline uint32_t
 327 mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 328                      struct rte_crypto_op *op, struct rte_mbuf *mbuf,
 329                      struct mlx5_wqe_dseg *klm)
 330 {
 331         uint32_t remain_len = op->sym->cipher.data.length;
 332         uint32_t nb_segs = mbuf->nb_segs;
 333         uint32_t klm_n = 1u;
 334
 335         /* First mbuf needs to take the cipher offset. */
 336         if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
 337                      op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 338                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 339                 return 0;
 340         }
 341         while (remain_len) {
 342                 nb_segs--;
 343                 mbuf = mbuf->next;
 344                 if (unlikely(mbuf == NULL || nb_segs == 0)) {
 345                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 346                         return 0;
 347                 }
 348                 if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
 349                                                  &remain_len) == UINT32_MAX)) {
 350                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 351                         return 0;
 352                 }
 353                 klm_n++;
 354         }
 355         return klm_n;
 356 }
 357
 358 static __rte_always_inline int
 359 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 360                          struct mlx5_crypto_qp *qp,
 361                          struct rte_crypto_op *op,
 362                          struct mlx5_umr_wqe *umr)
 363 {
 364         struct mlx5_crypto_session *sess = get_sym_session_private_data
 365                                 (op->sym->session, mlx5_crypto_driver_id);
 366         struct mlx5_wqe_cseg *cseg = &umr->ctr;
 367         struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
 368         struct mlx5_wqe_dseg *klms = &umr->kseg[0];
 369         struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
 370                                       RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
 371         uint32_t ds;
 372         bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 373         /* Set UMR WQE. */
 374         uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
 375                                    ipl ? op->sym->m_src : op->sym->m_dst, klms);
 376
 377         if (unlikely(klm_n == 0))
 378                 return 0;
 379         bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
 380         if (unlikely(!sess->bsp_res)) {
 381                 bsf->bsp_res = mlx5_crypto_get_block_size(op);
 382                 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
 383                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 384                         return 0;
 385                 }
 386         } else {
 387                 bsf->bsp_res = sess->bsp_res;
 388         }
 389         bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
 390         memcpy(bsf->xts_initial_tweak,
 391                rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
 392         bsf->res_dp = sess->dek_id;
 393         mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
 394         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
 395         qp->db_pi += priv->umr_wqe_stride;
 396         /* Set RDMA_WRITE WQE. */
 397         cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 398         klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 399         if (!ipl) {
 400                 klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
 401                                              klms);
 402                 if (unlikely(klm_n == 0))
 403                         return 0;
 404         } else {
 405                 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
 406         }
 407         ds = 2 + klm_n;
 408         cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 409         cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 410                                                         MLX5_OPCODE_RDMA_WRITE);
 411         ds = RTE_ALIGN(ds, 4);
 412         qp->db_pi += ds >> 2;
 413         /* Set NOP WQE if needed. */
 414         if (priv->max_rdmar_ds > ds) {
 415                 cseg += ds;
 416                 ds = priv->max_rdmar_ds - ds;
 417                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
 418                 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
 419                                                                MLX5_OPCODE_NOP);
 420                 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
 421         }
 422         qp->wqe = (uint8_t *)cseg;
 423         return 1;
 424 }
 425
 426 static __rte_always_inline void
 427 mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
 428 {
 429 #ifdef RTE_ARCH_64
 430         *priv->uar_addr = val;
 431 #else /* !RTE_ARCH_64 */
 432         rte_spinlock_lock(&priv->uar32_sl);
 433         *(volatile uint32_t *)priv->uar_addr = val;
 434         rte_io_wmb();
 435         *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
 436         rte_spinlock_unlock(&priv->uar32_sl);
 437 #endif
 438 }
 439
 440 static uint16_t
 441 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 442                           uint16_t nb_ops)
 443 {
 444         struct mlx5_crypto_qp *qp = queue_pair;
 445         struct mlx5_crypto_priv *priv = qp->priv;
 446         struct mlx5_umr_wqe *umr;
 447         struct rte_crypto_op *op;
 448         uint16_t mask = qp->entries_n - 1;
 449         uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
 450         uint32_t idx;
 451
 452         if (remain < nb_ops)
 453                 nb_ops = remain;
 454         else
 455                 remain = nb_ops;
 456         if (unlikely(remain == 0))
 457                 return 0;
 458         do {
 459                 idx = qp->pi & mask;
 460                 op = *ops++;
 461                 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 462                         priv->wqe_set_size * idx);
 463                 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
 464                         qp->stats.enqueue_err_count++;
 465                         if (remain != nb_ops) {
 466                                 qp->stats.enqueued_count -= remain;
 467                                 break;
 468                         }
 469                         return 0;
 470                 }
 471                 qp->ops[idx] = op;
 472                 qp->pi++;
 473         } while (--remain);
 474         qp->stats.enqueued_count += nb_ops;
 475         rte_io_wmb();
 476         qp->qp_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 477         rte_wmb();
 478         mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
 479         rte_wmb();
 480         return nb_ops;
 481 }
 482
 483 static __rte_noinline void
 484 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 485 {
 486         const uint32_t idx = qp->ci & (qp->entries_n - 1);
 487         volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
 488                                                         &qp->cq_obj.cqes[idx];
 489
 490         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 491         qp->stats.dequeue_err_count++;
 492         DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 493 }
 494
 495 static uint16_t
 496 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 497                           uint16_t nb_ops)
 498 {
 499         struct mlx5_crypto_qp *qp = queue_pair;
 500         volatile struct mlx5_cqe *restrict cqe;
 501         struct rte_crypto_op *restrict op;
 502         const unsigned int cq_size = qp->entries_n;
 503         const unsigned int mask = cq_size - 1;
 504         uint32_t idx;
 505         uint32_t next_idx = qp->ci & mask;
 506         const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
 507         uint16_t i = 0;
 508         int ret;
 509
 510         if (unlikely(max == 0))
 511                 return 0;
 512         do {
 513                 idx = next_idx;
 514                 next_idx = (qp->ci + 1) & mask;
 515                 op = qp->ops[idx];
 516                 cqe = &qp->cq_obj.cqes[idx];
 517                 ret = check_cqe(cqe, cq_size, qp->ci);
 518                 rte_io_rmb();
 519                 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 520                         if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
 521                                 mlx5_crypto_cqe_err_handle(qp, op);
 522                         break;
 523                 }
 524                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 525                 ops[i++] = op;
 526                 qp->ci++;
 527         } while (i < max);
 528         if (likely(i != 0)) {
 529                 rte_io_wmb();
 530                 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
 531                 qp->stats.dequeued_count += i;
 532         }
 533         return i;
 534 }
 535
 536 static void
 537 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 538 {
 539         uint32_t i;
 540
 541         for (i = 0 ; i < qp->entries_n; i++) {
 542                 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
 543                         i * priv->wqe_set_size);
 544                 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
 545                                                                      (cseg + 1);
 546                 struct mlx5_wqe_umr_bsf_seg *bsf =
 547                         (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
 548                                                        priv->umr_wqe_size)) - 1;
 549                 struct mlx5_wqe_rseg *rseg;
 550
 551                 /* Init UMR WQE. */
 552                 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
 553                                          (priv->umr_wqe_size / MLX5_WSEG_SIZE));
 554                 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
 555                                        MLX5_COMP_MODE_OFFSET);
 556                 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
 557                 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
 558                 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
 559                 ucseg->ko_to_bs = rte_cpu_to_be_32
 560                         ((RTE_ALIGN(priv->max_segs_num, 4u) <<
 561                          MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
 562                 bsf->keytag = priv->keytag;
 563                 /* Init RDMA WRITE WQE. */
 564                 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 565                 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
 566                                       MLX5_COMP_MODE_OFFSET) |
 567                                       MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
 568                 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
 569                 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
 570         }
 571 }
 572
 573 static int
 574 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
 575                                   struct mlx5_crypto_qp *qp)
 576 {
 577         struct mlx5_umr_wqe *umr;
 578         uint32_t i;
 579         struct mlx5_devx_mkey_attr attr = {
 580                 .pd = priv->cdev->pdn,
 581                 .umr_en = 1,
 582                 .crypto_en = 1,
 583                 .set_remote_rw = 1,
 584                 .klm_num = RTE_ALIGN(priv->max_segs_num, 4),
 585         };
 586
 587         for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
 588            i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
 589                 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
 590                 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
 591                 if (!qp->mkey[i])
 592                         goto error;
 593         }
 594         return 0;
 595 error:
 596         DRV_LOG(ERR, "Failed to allocate indirect mkey.");
 597         mlx5_crypto_indirect_mkeys_release(qp, i);
 598         return -1;
 599 }
 600
 601 static int
 602 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 603                              const struct rte_cryptodev_qp_conf *qp_conf,
 604                              int socket_id)
 605 {
 606         struct mlx5_crypto_priv *priv = dev->data->dev_private;
 607         struct mlx5_devx_qp_attr attr = {0};
 608         struct mlx5_crypto_qp *qp;
 609         uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 610         uint32_t ret;
 611         uint32_t alloc_size = sizeof(*qp);
 612         struct mlx5_devx_cq_attr cq_attr = {
 613                 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
 614         };
 615
 616         if (dev->data->queue_pairs[qp_id] != NULL)
 617                 mlx5_crypto_queue_pair_release(dev, qp_id);
 618         alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
 619         alloc_size += (sizeof(struct rte_crypto_op *) +
 620                        sizeof(struct mlx5_devx_obj *)) *
 621                        RTE_BIT32(log_nb_desc);
 622         qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 623                                 socket_id);
 624         if (qp == NULL) {
 625                 DRV_LOG(ERR, "Failed to allocate QP memory.");
 626                 rte_errno = ENOMEM;
 627                 return -rte_errno;
 628         }
 629         if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc,
 630                                 &cq_attr, socket_id) != 0) {
 631                 DRV_LOG(ERR, "Failed to create CQ.");
 632                 goto error;
 633         }
 634         attr.pd = priv->cdev->pdn;
 635         attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 636         attr.cqn = qp->cq_obj.cq->id;
 637         attr.rq_size = 0;
 638         attr.sq_size = RTE_BIT32(log_nb_desc);
 639         attr.ts_format =
 640                 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 641         ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, log_nb_desc,
 642                                   &attr, socket_id);
 643         if (ret) {
 644                 DRV_LOG(ERR, "Failed to create QP.");
 645                 goto error;
 646         }
 647         if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
 648                               priv->dev_config.socket_id) != 0) {
 649                 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 650                         (uint32_t)qp_id);
 651                 rte_errno = ENOMEM;
 652                 goto error;
 653         }
 654         /*
 655          * In Order to configure self loopback, when calling devx qp2rts the
 656          * remote QP id that is used is the id of the same QP.
 657          */
 658         if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id))
 659                 goto error;
 660         qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 661                                                            RTE_CACHE_LINE_SIZE);
 662         qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
 663         qp->entries_n = 1 << log_nb_desc;
 664         if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
 665                 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
 666                 rte_errno = ENOMEM;
 667                 goto error;
 668         }
 669         mlx5_crypto_qp_init(priv, qp);
 670         qp->priv = priv;
 671         dev->data->queue_pairs[qp_id] = qp;
 672         return 0;
 673 error:
 674         mlx5_crypto_qp_release(qp);
 675         return -1;
 676 }
 677
 678 static void
 679 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
 680                       struct rte_cryptodev_stats *stats)
 681 {
 682         int qp_id;
 683
 684         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 685                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 686
 687                 stats->enqueued_count += qp->stats.enqueued_count;
 688                 stats->dequeued_count += qp->stats.dequeued_count;
 689                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 690                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 691         }
 692 }
 693
 694 static void
 695 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
 696 {
 697         int qp_id;
 698
 699         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 700                 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
 701
 702                 memset(&qp->stats, 0, sizeof(qp->stats));
 703         }
 704 }
 705
 706 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 707         .dev_configure                  = mlx5_crypto_dev_configure,
 708         .dev_start                      = mlx5_crypto_dev_start,
 709         .dev_stop                       = mlx5_crypto_dev_stop,
 710         .dev_close                      = mlx5_crypto_dev_close,
 711         .dev_infos_get                  = mlx5_crypto_dev_infos_get,
 712         .stats_get                      = mlx5_crypto_stats_get,
 713         .stats_reset                    = mlx5_crypto_stats_reset,
 714         .queue_pair_setup               = mlx5_crypto_queue_pair_setup,
 715         .queue_pair_release             = mlx5_crypto_queue_pair_release,
 716         .sym_session_get_size           = mlx5_crypto_sym_session_get_size,
 717         .sym_session_configure          = mlx5_crypto_sym_session_configure,
 718         .sym_session_clear              = mlx5_crypto_sym_session_clear,
 719         .sym_get_raw_dp_ctx_size        = NULL,
 720         .sym_configure_raw_dp_ctx       = NULL,
 721 };
 722
 723 static void
 724 mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv)
 725 {
 726         if (priv->uar != NULL) {
 727                 mlx5_glue->devx_free_uar(priv->uar);
 728                 priv->uar = NULL;
 729         }
 730 }
 731
 732 static int
 733 mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv)
 734 {
 735         priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
 736         if (priv->uar)
 737                 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
 738         if (priv->uar == NULL || priv->uar_addr == NULL) {
 739                 rte_errno = errno;
 740                 DRV_LOG(ERR, "Failed to allocate UAR.");
 741                 return -1;
 742         }
 743         return 0;
 744 }
 745
 746
 747 static int
 748 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 749 {
 750         struct mlx5_crypto_devarg_params *devarg_prms = opaque;
 751         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 752         unsigned long tmp;
 753         FILE *file;
 754         int ret;
 755         int i;
 756
 757         if (strcmp(key, "class") == 0)
 758                 return 0;
 759         if (strcmp(key, "wcs_file") == 0) {
 760                 file = fopen(val, "rb");
 761                 if (file == NULL) {
 762                         rte_errno = ENOTSUP;
 763                         return -rte_errno;
 764                 }
 765                 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
 766                         ret = fscanf(file, "%02hhX", &attr->credential[i]);
 767                         if (ret <= 0) {
 768                                 fclose(file);
 769                                 DRV_LOG(ERR,
 770                                         "Failed to read credential from file.");
 771                                 rte_errno = EINVAL;
 772                                 return -rte_errno;
 773                         }
 774                 }
 775                 fclose(file);
 776                 devarg_prms->login_devarg = true;
 777                 return 0;
 778         }
 779         errno = 0;
 780         tmp = strtoul(val, NULL, 0);
 781         if (errno) {
 782                 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 783                 return -errno;
 784         }
 785         if (strcmp(key, "max_segs_num") == 0) {
 786                 if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
 787                         DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
 788                                 " be less than %d.",
 789                                 (uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
 790                         rte_errno = EINVAL;
 791                         return -rte_errno;
 792                 }
 793                 devarg_prms->max_segs_num = (uint32_t)tmp;
 794         } else if (strcmp(key, "import_kek_id") == 0) {
 795                 attr->session_import_kek_ptr = (uint32_t)tmp;
 796         } else if (strcmp(key, "credential_id") == 0) {
 797                 attr->credential_pointer = (uint32_t)tmp;
 798         } else if (strcmp(key, "keytag") == 0) {
 799                 devarg_prms->keytag = tmp;
 800         } else {
 801                 DRV_LOG(WARNING, "Invalid key %s.", key);
 802         }
 803         return 0;
 804 }
 805
 806 static int
 807 mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 808                           struct mlx5_crypto_devarg_params *devarg_prms)
 809 {
 810         struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 811         struct rte_kvargs *kvlist;
 812
 813         /* Default values. */
 814         attr->credential_pointer = 0;
 815         attr->session_import_kek_ptr = 0;
 816         devarg_prms->keytag = 0;
 817         devarg_prms->max_segs_num = 8;
 818         if (devargs == NULL) {
 819                 DRV_LOG(ERR,
 820         "No login devargs in order to enable crypto operations in the device.");
 821                 rte_errno = EINVAL;
 822                 return -1;
 823         }
 824         kvlist = rte_kvargs_parse(devargs->args, NULL);
 825         if (kvlist == NULL) {
 826                 DRV_LOG(ERR, "Failed to parse devargs.");
 827                 rte_errno = EINVAL;
 828                 return -1;
 829         }
 830         if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
 831                            devarg_prms) != 0) {
 832                 DRV_LOG(ERR, "Devargs handler function Failed.");
 833                 rte_kvargs_free(kvlist);
 834                 rte_errno = EINVAL;
 835                 return -1;
 836         }
 837         rte_kvargs_free(kvlist);
 838         if (devarg_prms->login_devarg == false) {
 839                 DRV_LOG(ERR,
 840         "No login credential devarg in order to enable crypto operations "
 841         "in the device.");
 842                 rte_errno = EINVAL;
 843                 return -1;
 844         }
 845         return 0;
 846 }
 847
 848 static int
 849 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 850 {
 851         struct rte_cryptodev *crypto_dev;
 852         struct mlx5_devx_obj *login;
 853         struct mlx5_crypto_priv *priv;
 854         struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 855         struct rte_cryptodev_pmd_init_params init_params = {
 856                 .name = "",
 857                 .private_data_size = sizeof(struct mlx5_crypto_priv),
 858                 .socket_id = cdev->dev->numa_node,
 859                 .max_nb_queue_pairs =
 860                                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 861         };
 862         const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
 863         uint16_t rdmw_wqe_size;
 864         int ret;
 865
 866         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 867                 DRV_LOG(ERR, "Non-primary process type is not supported.");
 868                 rte_errno = ENOTSUP;
 869                 return -rte_errno;
 870         }
 871         if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
 872                 DRV_LOG(ERR, "Not enough capabilities to support crypto "
 873                         "operations, maybe old FW/OFED version?");
 874                 rte_errno = ENOTSUP;
 875                 return -ENOTSUP;
 876         }
 877         ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms);
 878         if (ret) {
 879                 DRV_LOG(ERR, "Failed to parse devargs.");
 880                 return -rte_errno;
 881         }
 882         login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
 883                                                       &devarg_prms.login_attr);
 884         if (login == NULL) {
 885                 DRV_LOG(ERR, "Failed to configure login.");
 886                 return -rte_errno;
 887         }
 888         crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
 889                                               &init_params);
 890         if (crypto_dev == NULL) {
 891                 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
 892                 return -ENODEV;
 893         }
 894         DRV_LOG(INFO,
 895                 "Crypto device %s was created successfully.", ibdev_name);
 896         crypto_dev->dev_ops = &mlx5_crypto_ops;
 897         crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 898         crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 899         crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 900         crypto_dev->driver_id = mlx5_crypto_driver_id;
 901         priv = crypto_dev->data->dev_private;
 902         priv->cdev = cdev;
 903         priv->login_obj = login;
 904         priv->crypto_dev = crypto_dev;
 905         if (mlx5_crypto_uar_prepare(priv) != 0) {
 906                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 907                 return -1;
 908         }
 909         priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 910         priv->max_segs_num = devarg_prms.max_segs_num;
 911         priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
 912                              sizeof(struct mlx5_wqe_cseg) +
 913                              sizeof(struct mlx5_wqe_umr_cseg) +
 914                              sizeof(struct mlx5_wqe_mkey_cseg) +
 915                              RTE_ALIGN(priv->max_segs_num, 4) *
 916                              sizeof(struct mlx5_wqe_dseg);
 917         rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
 918                               sizeof(struct mlx5_wqe_dseg) *
 919                               (priv->max_segs_num <= 2 ? 2 : 2 +
 920                                RTE_ALIGN(priv->max_segs_num - 2, 4));
 921         priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
 922         priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
 923         priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 924         pthread_mutex_lock(&priv_list_lock);
 925         TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 926         pthread_mutex_unlock(&priv_list_lock);
 927
 928         rte_cryptodev_pmd_probing_finish(crypto_dev);
 929
 930         return 0;
 931 }
 932
 933 static int
 934 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
 935 {
 936         struct mlx5_crypto_priv *priv = NULL;
 937
 938         pthread_mutex_lock(&priv_list_lock);
 939         TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
 940                 if (priv->crypto_dev->device == cdev->dev)
 941                         break;
 942         if (priv)
 943                 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 944         pthread_mutex_unlock(&priv_list_lock);
 945         if (priv) {
 946                 mlx5_crypto_uar_release(priv);
 947                 rte_cryptodev_pmd_destroy(priv->crypto_dev);
 948                 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 949         }
 950         return 0;
 951 }
 952
 953 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
 954                 {
 955                         RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 956                                         PCI_DEVICE_ID_MELLANOX_CONNECTX6)
 957                 },
 958                 {
 959                         .vendor_id = 0
 960                 }
 961 };
 962
 963 static struct mlx5_class_driver mlx5_crypto_driver = {
 964         .drv_class = MLX5_CLASS_CRYPTO,
 965         .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
 966         .id_table = mlx5_crypto_pci_id_map,
 967         .probe = mlx5_crypto_dev_probe,
 968         .remove = mlx5_crypto_dev_remove,
 969 };
 970
 971 RTE_INIT(rte_mlx5_crypto_init)
 972 {
 973         pthread_mutex_init(&priv_list_lock, NULL);
 974         mlx5_common_init();
 975         if (mlx5_glue != NULL)
 976                 mlx5_class_driver_register(&mlx5_crypto_driver);
 977 }
 978
 979 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
 980                                mlx5_crypto_driver_id);
 981
 982 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
 983 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
 984 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
 985 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");