drivers/crypto/octeontx2/otx2_cryptodev_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include <rte_cryptodev_pmd.h>
   8 #include <rte_errno.h>
   9 #include <rte_ethdev.h>
  10
  11 #include "otx2_cryptodev.h"
  12 #include "otx2_cryptodev_capabilities.h"
  13 #include "otx2_cryptodev_hw_access.h"
  14 #include "otx2_cryptodev_mbox.h"
  15 #include "otx2_cryptodev_ops.h"
  16 #include "otx2_mbox.h"
  17 #include "otx2_sec_idev.h"
  18
  19 #include "cpt_hw_types.h"
  20 #include "cpt_pmd_logs.h"
  21 #include "cpt_pmd_ops_helper.h"
  22 #include "cpt_ucode.h"
  23 #include "cpt_ucode_asym.h"
  24
  25 #define METABUF_POOL_CACHE_SIZE 512
  26
  27 static uint64_t otx2_fpm_iova[CPT_EC_ID_PMAX];
  28
  29 /* Forward declarations */
  30
  31 static int
  32 otx2_cpt_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id);
  33
  34 static void
  35 qp_memzone_name_get(char *name, int size, int dev_id, int qp_id)
  36 {
  37         snprintf(name, size, "otx2_cpt_lf_mem_%u:%u", dev_id, qp_id);
  38 }
  39
  40 static int
  41 otx2_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev,
  42                                 struct otx2_cpt_qp *qp, uint8_t qp_id,
  43                                 int nb_elements)
  44 {
  45         char mempool_name[RTE_MEMPOOL_NAMESIZE];
  46         struct cpt_qp_meta_info *meta_info;
  47         struct rte_mempool *pool;
  48         int ret, max_mlen;
  49         int asym_mlen = 0;
  50         int lb_mlen = 0;
  51         int sg_mlen = 0;
  52
  53         if (dev->feature_flags & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {
  54
  55                 /* Get meta len for scatter gather mode */
  56                 sg_mlen = cpt_pmd_ops_helper_get_mlen_sg_mode();
  57
  58                 /* Extra 32B saved for future considerations */
  59                 sg_mlen += 4 * sizeof(uint64_t);
  60
  61                 /* Get meta len for linear buffer (direct) mode */
  62                 lb_mlen = cpt_pmd_ops_helper_get_mlen_direct_mode();
  63
  64                 /* Extra 32B saved for future considerations */
  65                 lb_mlen += 4 * sizeof(uint64_t);
  66         }
  67
  68         if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {
  69
  70                 /* Get meta len required for asymmetric operations */
  71                 asym_mlen = cpt_pmd_ops_helper_asym_get_mlen();
  72         }
  73
  74         /*
  75          * Check max requirement for meta buffer to
  76          * support crypto op of any type (sym/asym).
  77          */
  78         max_mlen = RTE_MAX(RTE_MAX(lb_mlen, sg_mlen), asym_mlen);
  79
  80         /* Allocate mempool */
  81
  82         snprintf(mempool_name, RTE_MEMPOOL_NAMESIZE, "otx2_cpt_mb_%u:%u",
  83                  dev->data->dev_id, qp_id);
  84
  85         pool = rte_mempool_create_empty(mempool_name, nb_elements, max_mlen,
  86                                         METABUF_POOL_CACHE_SIZE, 0,
  87                                         rte_socket_id(), 0);
  88
  89         if (pool == NULL) {
  90                 CPT_LOG_ERR("Could not create mempool for metabuf");
  91                 return rte_errno;
  92         }
  93
  94         ret = rte_mempool_set_ops_byname(pool, RTE_MBUF_DEFAULT_MEMPOOL_OPS,
  95                                          NULL);
  96         if (ret) {
  97                 CPT_LOG_ERR("Could not set mempool ops");
  98                 goto mempool_free;
  99         }
 100
 101         ret = rte_mempool_populate_default(pool);
 102         if (ret <= 0) {
 103                 CPT_LOG_ERR("Could not populate metabuf pool");
 104                 goto mempool_free;
 105         }
 106
 107         meta_info = &qp->meta_info;
 108
 109         meta_info->pool = pool;
 110         meta_info->lb_mlen = lb_mlen;
 111         meta_info->sg_mlen = sg_mlen;
 112
 113         return 0;
 114
 115 mempool_free:
 116         rte_mempool_free(pool);
 117         return ret;
 118 }
 119
 120 static void
 121 otx2_cpt_metabuf_mempool_destroy(struct otx2_cpt_qp *qp)
 122 {
 123         struct cpt_qp_meta_info *meta_info = &qp->meta_info;
 124
 125         rte_mempool_free(meta_info->pool);
 126
 127         meta_info->pool = NULL;
 128         meta_info->lb_mlen = 0;
 129         meta_info->sg_mlen = 0;
 130 }
 131
 132 static int
 133 otx2_cpt_qp_inline_cfg(const struct rte_cryptodev *dev, struct otx2_cpt_qp *qp)
 134 {
 135         static rte_atomic16_t port_offset = RTE_ATOMIC16_INIT(-1);
 136         uint16_t port_id, nb_ethport = rte_eth_dev_count_avail();
 137         int i, ret;
 138
 139         for (i = 0; i < nb_ethport; i++) {
 140                 port_id = rte_atomic16_add_return(&port_offset, 1) % nb_ethport;
 141                 if (otx2_eth_dev_is_sec_capable(&rte_eth_devices[port_id]))
 142                         break;
 143         }
 144
 145         if (i >= nb_ethport)
 146                 return 0;
 147
 148         ret = otx2_cpt_qp_ethdev_bind(dev, qp, port_id);
 149         if (ret)
 150                 return ret;
 151
 152         /* Publish inline Tx QP to eth dev security */
 153         ret = otx2_sec_idev_tx_cpt_qp_add(port_id, qp);
 154         if (ret)
 155                 return ret;
 156
 157         return 0;
 158 }
 159
 160 static struct otx2_cpt_qp *
 161 otx2_cpt_qp_create(const struct rte_cryptodev *dev, uint16_t qp_id,
 162                    uint8_t group)
 163 {
 164         struct otx2_cpt_vf *vf = dev->data->dev_private;
 165         uint64_t pg_sz = sysconf(_SC_PAGESIZE);
 166         const struct rte_memzone *lf_mem;
 167         uint32_t len, iq_len, size_div40;
 168         char name[RTE_MEMZONE_NAMESIZE];
 169         uint64_t used_len, iova;
 170         struct otx2_cpt_qp *qp;
 171         uint64_t lmtline;
 172         uint8_t *va;
 173         int ret;
 174
 175         /* Allocate queue pair */
 176         qp = rte_zmalloc_socket("OCTEON TX2 Crypto PMD Queue Pair", sizeof(*qp),
 177                                 OTX2_ALIGN, 0);
 178         if (qp == NULL) {
 179                 CPT_LOG_ERR("Could not allocate queue pair");
 180                 return NULL;
 181         }
 182
 183         iq_len = OTX2_CPT_IQ_LEN;
 184
 185         /*
 186          * Queue size must be a multiple of 40 and effective queue size to
 187          * software is (size_div40 - 1) * 40
 188          */
 189         size_div40 = (iq_len + 40 - 1) / 40 + 1;
 190
 191         /* For pending queue */
 192         len = iq_len * RTE_ALIGN(sizeof(struct rid), 8);
 193
 194         /* Space for instruction group memory */
 195         len += size_div40 * 16;
 196
 197         /* So that instruction queues start as pg size aligned */
 198         len = RTE_ALIGN(len, pg_sz);
 199
 200         /* For instruction queues */
 201         len += OTX2_CPT_IQ_LEN * sizeof(union cpt_inst_s);
 202
 203         /* Wastage after instruction queues */
 204         len = RTE_ALIGN(len, pg_sz);
 205
 206         qp_memzone_name_get(name, RTE_MEMZONE_NAMESIZE, dev->data->dev_id,
 207                             qp_id);
 208
 209         lf_mem = rte_memzone_reserve_aligned(name, len, vf->otx2_dev.node,
 210                         RTE_MEMZONE_SIZE_HINT_ONLY | RTE_MEMZONE_256MB,
 211                         RTE_CACHE_LINE_SIZE);
 212         if (lf_mem == NULL) {
 213                 CPT_LOG_ERR("Could not allocate reserved memzone");
 214                 goto qp_free;
 215         }
 216
 217         va = lf_mem->addr;
 218         iova = lf_mem->iova;
 219
 220         memset(va, 0, len);
 221
 222         ret = otx2_cpt_metabuf_mempool_create(dev, qp, qp_id, iq_len);
 223         if (ret) {
 224                 CPT_LOG_ERR("Could not create mempool for metabuf");
 225                 goto lf_mem_free;
 226         }
 227
 228         /* Initialize pending queue */
 229         qp->pend_q.rid_queue = (struct rid *)va;
 230         qp->pend_q.enq_tail = 0;
 231         qp->pend_q.deq_head = 0;
 232         qp->pend_q.pending_count = 0;
 233
 234         used_len = iq_len * RTE_ALIGN(sizeof(struct rid), 8);
 235         used_len += size_div40 * 16;
 236         used_len = RTE_ALIGN(used_len, pg_sz);
 237         iova += used_len;
 238
 239         qp->iq_dma_addr = iova;
 240         qp->id = qp_id;
 241         qp->base = OTX2_CPT_LF_BAR2(vf, qp_id);
 242
 243         lmtline = vf->otx2_dev.bar2 +
 244                   (RVU_BLOCK_ADDR_LMT << 20 | qp_id << 12) +
 245                   OTX2_LMT_LF_LMTLINE(0);
 246
 247         qp->lmtline = (void *)lmtline;
 248
 249         qp->lf_nq_reg = qp->base + OTX2_CPT_LF_NQ(0);
 250
 251         ret = otx2_sec_idev_tx_cpt_qp_remove(qp);
 252         if (ret && (ret != -ENOENT)) {
 253                 CPT_LOG_ERR("Could not delete inline configuration");
 254                 goto mempool_destroy;
 255         }
 256
 257         otx2_cpt_iq_disable(qp);
 258
 259         ret = otx2_cpt_qp_inline_cfg(dev, qp);
 260         if (ret) {
 261                 CPT_LOG_ERR("Could not configure queue for inline IPsec");
 262                 goto mempool_destroy;
 263         }
 264
 265         ret = otx2_cpt_iq_enable(dev, qp, group, OTX2_CPT_QUEUE_HI_PRIO,
 266                                  size_div40);
 267         if (ret) {
 268                 CPT_LOG_ERR("Could not enable instruction queue");
 269                 goto mempool_destroy;
 270         }
 271
 272         return qp;
 273
 274 mempool_destroy:
 275         otx2_cpt_metabuf_mempool_destroy(qp);
 276 lf_mem_free:
 277         rte_memzone_free(lf_mem);
 278 qp_free:
 279         rte_free(qp);
 280         return NULL;
 281 }
 282
 283 static int
 284 otx2_cpt_qp_destroy(const struct rte_cryptodev *dev, struct otx2_cpt_qp *qp)
 285 {
 286         const struct rte_memzone *lf_mem;
 287         char name[RTE_MEMZONE_NAMESIZE];
 288         int ret;
 289
 290         ret = otx2_sec_idev_tx_cpt_qp_remove(qp);
 291         if (ret && (ret != -ENOENT)) {
 292                 CPT_LOG_ERR("Could not delete inline configuration");
 293                 return ret;
 294         }
 295
 296         otx2_cpt_iq_disable(qp);
 297
 298         otx2_cpt_metabuf_mempool_destroy(qp);
 299
 300         qp_memzone_name_get(name, RTE_MEMZONE_NAMESIZE, dev->data->dev_id,
 301                             qp->id);
 302
 303         lf_mem = rte_memzone_lookup(name);
 304
 305         ret = rte_memzone_free(lf_mem);
 306         if (ret)
 307                 return ret;
 308
 309         rte_free(qp);
 310
 311         return 0;
 312 }
 313
 314 static int
 315 sym_xform_verify(struct rte_crypto_sym_xform *xform)
 316 {
 317         if (xform->next) {
 318                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 319                     xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 320                     xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 321                         return -ENOTSUP;
 322
 323                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 324                     xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
 325                     xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 326                         return -ENOTSUP;
 327
 328                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 329                     xform->cipher.algo == RTE_CRYPTO_CIPHER_3DES_CBC &&
 330                     xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 331                     xform->next->auth.algo == RTE_CRYPTO_AUTH_SHA1)
 332                         return -ENOTSUP;
 333
 334                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 335                     xform->auth.algo == RTE_CRYPTO_AUTH_SHA1 &&
 336                     xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 337                     xform->next->cipher.algo == RTE_CRYPTO_CIPHER_3DES_CBC)
 338                         return -ENOTSUP;
 339
 340         } else {
 341                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 342                     xform->auth.algo == RTE_CRYPTO_AUTH_NULL &&
 343                     xform->auth.op == RTE_CRYPTO_AUTH_OP_VERIFY)
 344                         return -ENOTSUP;
 345         }
 346         return 0;
 347 }
 348
 349 static int
 350 sym_session_configure(int driver_id, struct rte_crypto_sym_xform *xform,
 351                       struct rte_cryptodev_sym_session *sess,
 352                       struct rte_mempool *pool)
 353 {
 354         struct cpt_sess_misc *misc;
 355         void *priv;
 356         int ret;
 357
 358         ret = sym_xform_verify(xform);
 359         if (unlikely(ret))
 360                 return ret;
 361
 362         if (unlikely(rte_mempool_get(pool, &priv))) {
 363                 CPT_LOG_ERR("Could not allocate session private data");
 364                 return -ENOMEM;
 365         }
 366
 367         misc = priv;
 368
 369         for ( ; xform != NULL; xform = xform->next) {
 370                 switch (xform->type) {
 371                 case RTE_CRYPTO_SYM_XFORM_AEAD:
 372                         ret = fill_sess_aead(xform, misc);
 373                         break;
 374                 case RTE_CRYPTO_SYM_XFORM_CIPHER:
 375                         ret = fill_sess_cipher(xform, misc);
 376                         break;
 377                 case RTE_CRYPTO_SYM_XFORM_AUTH:
 378                         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC)
 379                                 ret = fill_sess_gmac(xform, misc);
 380                         else
 381                                 ret = fill_sess_auth(xform, misc);
 382                         break;
 383                 default:
 384                         ret = -1;
 385                 }
 386
 387                 if (ret)
 388                         goto priv_put;
 389         }
 390
 391         set_sym_session_private_data(sess, driver_id, misc);
 392
 393         misc->ctx_dma_addr = rte_mempool_virt2iova(misc) +
 394                              sizeof(struct cpt_sess_misc);
 395
 396         /*
 397          * IE engines support IPsec operations
 398          * SE engines support IPsec operations and Air-Crypto operations
 399          */
 400         if (misc->zsk_flag)
 401                 misc->egrp = OTX2_CPT_EGRP_SE;
 402         else
 403                 misc->egrp = OTX2_CPT_EGRP_SE_IE;
 404
 405         return 0;
 406
 407 priv_put:
 408         rte_mempool_put(pool, priv);
 409
 410         return -ENOTSUP;
 411 }
 412
 413 static void
 414 sym_session_clear(int driver_id, struct rte_cryptodev_sym_session *sess)
 415 {
 416         void *priv = get_sym_session_private_data(sess, driver_id);
 417         struct rte_mempool *pool;
 418
 419         if (priv == NULL)
 420                 return;
 421
 422         memset(priv, 0, cpt_get_session_size());
 423
 424         pool = rte_mempool_from_obj(priv);
 425
 426         set_sym_session_private_data(sess, driver_id, NULL);
 427
 428         rte_mempool_put(pool, priv);
 429 }
 430
 431 static __rte_always_inline int32_t __rte_hot
 432 otx2_cpt_enqueue_req(const struct otx2_cpt_qp *qp,
 433                      struct pending_queue *pend_q,
 434                      struct cpt_request_info *req)
 435 {
 436         void *lmtline = qp->lmtline;
 437         union cpt_inst_s inst;
 438         uint64_t lmt_status;
 439
 440         if (unlikely(pend_q->pending_count >= OTX2_CPT_DEFAULT_CMD_QLEN))
 441                 return -EAGAIN;
 442
 443         inst.u[0] = 0;
 444         inst.s9x.res_addr = req->comp_baddr;
 445         inst.u[2] = 0;
 446         inst.u[3] = 0;
 447
 448         inst.s9x.ei0 = req->ist.ei0;
 449         inst.s9x.ei1 = req->ist.ei1;
 450         inst.s9x.ei2 = req->ist.ei2;
 451         inst.s9x.ei3 = req->ist.ei3;
 452
 453         req->time_out = rte_get_timer_cycles() +
 454                         DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
 455
 456         do {
 457                 /* Copy CPT command to LMTLINE */
 458                 memcpy(lmtline, &inst, sizeof(inst));
 459
 460                 /*
 461                  * Make sure compiler does not reorder memcpy and ldeor.
 462                  * LMTST transactions are always flushed from the write
 463                  * buffer immediately, a DMB is not required to push out
 464                  * LMTSTs.
 465                  */
 466                 rte_cio_wmb();
 467                 lmt_status = otx2_lmt_submit(qp->lf_nq_reg);
 468         } while (lmt_status == 0);
 469
 470         pend_q->rid_queue[pend_q->enq_tail].rid = (uintptr_t)req;
 471
 472         /* We will use soft queue length here to limit requests */
 473         MOD_INC(pend_q->enq_tail, OTX2_CPT_DEFAULT_CMD_QLEN);
 474         pend_q->pending_count += 1;
 475
 476         return 0;
 477 }
 478
 479 static __rte_always_inline int32_t __rte_hot
 480 otx2_cpt_enqueue_asym(struct otx2_cpt_qp *qp,
 481                       struct rte_crypto_op *op,
 482                       struct pending_queue *pend_q)
 483 {
 484         struct cpt_qp_meta_info *minfo = &qp->meta_info;
 485         struct rte_crypto_asym_op *asym_op = op->asym;
 486         struct asym_op_params params = {0};
 487         struct cpt_asym_sess_misc *sess;
 488         vq_cmd_word3_t *w3;
 489         uintptr_t *cop;
 490         void *mdata;
 491         int ret;
 492
 493         if (unlikely(rte_mempool_get(minfo->pool, &mdata) < 0)) {
 494                 CPT_LOG_ERR("Could not allocate meta buffer for request");
 495                 return -ENOMEM;
 496         }
 497
 498         sess = get_asym_session_private_data(asym_op->session,
 499                                              otx2_cryptodev_driver_id);
 500
 501         /* Store IO address of the mdata to meta_buf */
 502         params.meta_buf = rte_mempool_virt2iova(mdata);
 503
 504         cop = mdata;
 505         cop[0] = (uintptr_t)mdata;
 506         cop[1] = (uintptr_t)op;
 507         cop[2] = cop[3] = 0ULL;
 508
 509         params.req = RTE_PTR_ADD(cop, 4 * sizeof(uintptr_t));
 510         params.req->op = cop;
 511
 512         /* Adjust meta_buf to point to end of cpt_request_info structure */
 513         params.meta_buf += (4 * sizeof(uintptr_t)) +
 514                             sizeof(struct cpt_request_info);
 515         switch (sess->xfrm_type) {
 516         case RTE_CRYPTO_ASYM_XFORM_MODEX:
 517                 ret = cpt_modex_prep(&params, &sess->mod_ctx);
 518                 if (unlikely(ret))
 519                         goto req_fail;
 520                 break;
 521         case RTE_CRYPTO_ASYM_XFORM_RSA:
 522                 ret = cpt_enqueue_rsa_op(op, &params, sess);
 523                 if (unlikely(ret))
 524                         goto req_fail;
 525                 break;
 526         case RTE_CRYPTO_ASYM_XFORM_ECDSA:
 527                 ret = cpt_enqueue_ecdsa_op(op, &params, sess, otx2_fpm_iova);
 528                 if (unlikely(ret))
 529                         goto req_fail;
 530                 break;
 531         case RTE_CRYPTO_ASYM_XFORM_ECPM:
 532                 ret = cpt_ecpm_prep(&asym_op->ecpm, &params,
 533                                     sess->ec_ctx.curveid);
 534                 if (unlikely(ret))
 535                         goto req_fail;
 536                 break;
 537         default:
 538                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 539                 ret = -EINVAL;
 540                 goto req_fail;
 541         }
 542
 543         /* Set engine group of AE */
 544         w3 = (vq_cmd_word3_t *)&params.req->ist.ei3;
 545         w3->s.grp = OTX2_CPT_EGRP_AE;
 546
 547         ret = otx2_cpt_enqueue_req(qp, pend_q, params.req);
 548
 549         if (unlikely(ret)) {
 550                 CPT_LOG_DP_ERR("Could not enqueue crypto req");
 551                 goto req_fail;
 552         }
 553
 554         return 0;
 555
 556 req_fail:
 557         free_op_meta(mdata, minfo->pool);
 558
 559         return ret;
 560 }
 561
 562 static __rte_always_inline int __rte_hot
 563 otx2_cpt_enqueue_sym(struct otx2_cpt_qp *qp, struct rte_crypto_op *op,
 564                      struct pending_queue *pend_q)
 565 {
 566         struct rte_crypto_sym_op *sym_op = op->sym;
 567         struct cpt_request_info *req;
 568         struct cpt_sess_misc *sess;
 569         vq_cmd_word3_t *w3;
 570         uint64_t cpt_op;
 571         void *mdata;
 572         int ret;
 573
 574         sess = get_sym_session_private_data(sym_op->session,
 575                                             otx2_cryptodev_driver_id);
 576
 577         cpt_op = sess->cpt_op;
 578
 579         if (cpt_op & CPT_OP_CIPHER_MASK)
 580                 ret = fill_fc_params(op, sess, &qp->meta_info, &mdata,
 581                                      (void **)&req);
 582         else
 583                 ret = fill_digest_params(op, sess, &qp->meta_info, &mdata,
 584                                          (void **)&req);
 585
 586         if (unlikely(ret)) {
 587                 CPT_LOG_DP_ERR("Crypto req : op %p, cpt_op 0x%x ret 0x%x",
 588                                 op, (unsigned int)cpt_op, ret);
 589                 return ret;
 590         }
 591
 592         w3 = ((vq_cmd_word3_t *)(&req->ist.ei3));
 593         w3->s.grp = sess->egrp;
 594
 595         ret = otx2_cpt_enqueue_req(qp, pend_q, req);
 596
 597         if (unlikely(ret)) {
 598                 /* Free buffer allocated by fill params routines */
 599                 free_op_meta(mdata, qp->meta_info.pool);
 600         }
 601
 602         return ret;
 603 }
 604
 605 static __rte_always_inline int __rte_hot
 606 otx2_cpt_enqueue_sym_sessless(struct otx2_cpt_qp *qp, struct rte_crypto_op *op,
 607                               struct pending_queue *pend_q)
 608 {
 609         const int driver_id = otx2_cryptodev_driver_id;
 610         struct rte_crypto_sym_op *sym_op = op->sym;
 611         struct rte_cryptodev_sym_session *sess;
 612         int ret;
 613
 614         /* Create temporary session */
 615
 616         if (rte_mempool_get(qp->sess_mp, (void **)&sess))
 617                 return -ENOMEM;
 618
 619         ret = sym_session_configure(driver_id, sym_op->xform, sess,
 620                                     qp->sess_mp_priv);
 621         if (ret)
 622                 goto sess_put;
 623
 624         sym_op->session = sess;
 625
 626         ret = otx2_cpt_enqueue_sym(qp, op, pend_q);
 627
 628         if (unlikely(ret))
 629                 goto priv_put;
 630
 631         return 0;
 632
 633 priv_put:
 634         sym_session_clear(driver_id, sess);
 635 sess_put:
 636         rte_mempool_put(qp->sess_mp, sess);
 637         return ret;
 638 }
 639
 640 static uint16_t
 641 otx2_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 642 {
 643         uint16_t nb_allowed, count = 0;
 644         struct otx2_cpt_qp *qp = qptr;
 645         struct pending_queue *pend_q;
 646         struct rte_crypto_op *op;
 647         int ret;
 648
 649         pend_q = &qp->pend_q;
 650
 651         nb_allowed = OTX2_CPT_DEFAULT_CMD_QLEN - pend_q->pending_count;
 652         if (nb_ops > nb_allowed)
 653                 nb_ops = nb_allowed;
 654
 655         for (count = 0; count < nb_ops; count++) {
 656                 op = ops[count];
 657                 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 658                         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
 659                                 ret = otx2_cpt_enqueue_sym(qp, op, pend_q);
 660                         else
 661                                 ret = otx2_cpt_enqueue_sym_sessless(qp, op,
 662                                                                     pend_q);
 663                 } else if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
 664                         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
 665                                 ret = otx2_cpt_enqueue_asym(qp, op, pend_q);
 666                         else
 667                                 break;
 668                 } else
 669                         break;
 670
 671                 if (unlikely(ret))
 672                         break;
 673         }
 674
 675         return count;
 676 }
 677
 678 static __rte_always_inline void
 679 otx2_cpt_asym_rsa_op(struct rte_crypto_op *cop, struct cpt_request_info *req,
 680                      struct rte_crypto_rsa_xform *rsa_ctx)
 681 {
 682         struct rte_crypto_rsa_op_param *rsa = &cop->asym->rsa;
 683
 684         switch (rsa->op_type) {
 685         case RTE_CRYPTO_ASYM_OP_ENCRYPT:
 686                 rsa->cipher.length = rsa_ctx->n.length;
 687                 memcpy(rsa->cipher.data, req->rptr, rsa->cipher.length);
 688                 break;
 689         case RTE_CRYPTO_ASYM_OP_DECRYPT:
 690                 if (rsa->pad == RTE_CRYPTO_RSA_PADDING_NONE) {
 691                         rsa->message.length = rsa_ctx->n.length;
 692                         memcpy(rsa->message.data, req->rptr,
 693                                rsa->message.length);
 694                 } else {
 695                         /* Get length of decrypted output */
 696                         rsa->message.length = rte_cpu_to_be_16
 697                                              (*((uint16_t *)req->rptr));
 698                         /*
 699                          * Offset output data pointer by length field
 700                          * (2 bytes) and copy decrypted data.
 701                          */
 702                         memcpy(rsa->message.data, req->rptr + 2,
 703                                rsa->message.length);
 704                 }
 705                 break;
 706         case RTE_CRYPTO_ASYM_OP_SIGN:
 707                 rsa->sign.length = rsa_ctx->n.length;
 708                 memcpy(rsa->sign.data, req->rptr, rsa->sign.length);
 709                 break;
 710         case RTE_CRYPTO_ASYM_OP_VERIFY:
 711                 if (rsa->pad == RTE_CRYPTO_RSA_PADDING_NONE) {
 712                         rsa->sign.length = rsa_ctx->n.length;
 713                         memcpy(rsa->sign.data, req->rptr, rsa->sign.length);
 714                 } else {
 715                         /* Get length of signed output */
 716                         rsa->sign.length = rte_cpu_to_be_16
 717                                           (*((uint16_t *)req->rptr));
 718                         /*
 719                          * Offset output data pointer by length field
 720                          * (2 bytes) and copy signed data.
 721                          */
 722                         memcpy(rsa->sign.data, req->rptr + 2,
 723                                rsa->sign.length);
 724                 }
 725                 if (memcmp(rsa->sign.data, rsa->message.data,
 726                            rsa->message.length)) {
 727                         CPT_LOG_DP_ERR("RSA verification failed");
 728                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 729                 }
 730                 break;
 731         default:
 732                 CPT_LOG_DP_DEBUG("Invalid RSA operation type");
 733                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 734                 break;
 735         }
 736 }
 737
 738 static __rte_always_inline void
 739 otx2_cpt_asym_dequeue_ecdsa_op(struct rte_crypto_ecdsa_op_param *ecdsa,
 740                                struct cpt_request_info *req,
 741                                struct cpt_asym_ec_ctx *ec)
 742 {
 743         int prime_len = ec_grp[ec->curveid].prime.length;
 744
 745         if (ecdsa->op_type == RTE_CRYPTO_ASYM_OP_VERIFY)
 746                 return;
 747
 748         /* Separate out sign r and s components */
 749         memcpy(ecdsa->r.data, req->rptr, prime_len);
 750         memcpy(ecdsa->s.data, req->rptr + ROUNDUP8(prime_len), prime_len);
 751         ecdsa->r.length = prime_len;
 752         ecdsa->s.length = prime_len;
 753 }
 754
 755 static __rte_always_inline void
 756 otx2_cpt_asym_dequeue_ecpm_op(struct rte_crypto_ecpm_op_param *ecpm,
 757                              struct cpt_request_info *req,
 758                              struct cpt_asym_ec_ctx *ec)
 759 {
 760         int prime_len = ec_grp[ec->curveid].prime.length;
 761
 762         memcpy(ecpm->r.x.data, req->rptr, prime_len);
 763         memcpy(ecpm->r.y.data, req->rptr + ROUNDUP8(prime_len), prime_len);
 764         ecpm->r.x.length = prime_len;
 765         ecpm->r.y.length = prime_len;
 766 }
 767
 768 static void
 769 otx2_cpt_asym_post_process(struct rte_crypto_op *cop,
 770                            struct cpt_request_info *req)
 771 {
 772         struct rte_crypto_asym_op *op = cop->asym;
 773         struct cpt_asym_sess_misc *sess;
 774
 775         sess = get_asym_session_private_data(op->session,
 776                                              otx2_cryptodev_driver_id);
 777
 778         switch (sess->xfrm_type) {
 779         case RTE_CRYPTO_ASYM_XFORM_RSA:
 780                 otx2_cpt_asym_rsa_op(cop, req, &sess->rsa_ctx);
 781                 break;
 782         case RTE_CRYPTO_ASYM_XFORM_MODEX:
 783                 op->modex.result.length = sess->mod_ctx.modulus.length;
 784                 memcpy(op->modex.result.data, req->rptr,
 785                        op->modex.result.length);
 786                 break;
 787         case RTE_CRYPTO_ASYM_XFORM_ECDSA:
 788                 otx2_cpt_asym_dequeue_ecdsa_op(&op->ecdsa, req, &sess->ec_ctx);
 789                 break;
 790         case RTE_CRYPTO_ASYM_XFORM_ECPM:
 791                 otx2_cpt_asym_dequeue_ecpm_op(&op->ecpm, req, &sess->ec_ctx);
 792                 break;
 793         default:
 794                 CPT_LOG_DP_DEBUG("Invalid crypto xform type");
 795                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 796                 break;
 797         }
 798 }
 799
 800 static inline void
 801 otx2_cpt_dequeue_post_process(struct otx2_cpt_qp *qp, struct rte_crypto_op *cop,
 802                               uintptr_t *rsp, uint8_t cc)
 803 {
 804         if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 805                 if (likely(cc == NO_ERR)) {
 806                         /* Verify authentication data if required */
 807                         if (unlikely(rsp[2]))
 808                                 compl_auth_verify(cop, (uint8_t *)rsp[2],
 809                                                  rsp[3]);
 810                         else
 811                                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 812                 } else {
 813                         if (cc == ERR_GC_ICV_MISCOMPARE)
 814                                 cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 815                         else
 816                                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 817                 }
 818
 819                 if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
 820                         sym_session_clear(otx2_cryptodev_driver_id,
 821                                           cop->sym->session);
 822                         rte_mempool_put(qp->sess_mp, cop->sym->session);
 823                         cop->sym->session = NULL;
 824                 }
 825         }
 826
 827         if (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
 828                 if (likely(cc == NO_ERR)) {
 829                         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 830                         /*
 831                          * Pass cpt_req_info stored in metabuf during
 832                          * enqueue.
 833                          */
 834                         rsp = RTE_PTR_ADD(rsp, 4 * sizeof(uintptr_t));
 835                         otx2_cpt_asym_post_process(cop,
 836                                         (struct cpt_request_info *)rsp);
 837                 } else
 838                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 839         }
 840 }
 841
 842 static __rte_always_inline uint8_t
 843 otx2_cpt_compcode_get(struct cpt_request_info *req)
 844 {
 845         volatile struct cpt_res_s_9s *res;
 846         uint8_t ret;
 847
 848         res = (volatile struct cpt_res_s_9s *)req->completion_addr;
 849
 850         if (unlikely(res->compcode == CPT_9X_COMP_E_NOTDONE)) {
 851                 if (rte_get_timer_cycles() < req->time_out)
 852                         return ERR_REQ_PENDING;
 853
 854                 CPT_LOG_DP_ERR("Request timed out");
 855                 return ERR_REQ_TIMEOUT;
 856         }
 857
 858         if (likely(res->compcode == CPT_9X_COMP_E_GOOD)) {
 859                 ret = NO_ERR;
 860                 if (unlikely(res->uc_compcode)) {
 861                         ret = res->uc_compcode;
 862                         CPT_LOG_DP_DEBUG("Request failed with microcode error");
 863                         CPT_LOG_DP_DEBUG("MC completion code 0x%x",
 864                                          res->uc_compcode);
 865                 }
 866         } else {
 867                 CPT_LOG_DP_DEBUG("HW completion code 0x%x", res->compcode);
 868
 869                 ret = res->compcode;
 870                 switch (res->compcode) {
 871                 case CPT_9X_COMP_E_INSTERR:
 872                         CPT_LOG_DP_ERR("Request failed with instruction error");
 873                         break;
 874                 case CPT_9X_COMP_E_FAULT:
 875                         CPT_LOG_DP_ERR("Request failed with DMA fault");
 876                         break;
 877                 case CPT_9X_COMP_E_HWERR:
 878                         CPT_LOG_DP_ERR("Request failed with hardware error");
 879                         break;
 880                 default:
 881                         CPT_LOG_DP_ERR("Request failed with unknown completion code");
 882                 }
 883         }
 884
 885         return ret;
 886 }
 887
 888 static uint16_t
 889 otx2_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 890 {
 891         int i, nb_pending, nb_completed;
 892         struct otx2_cpt_qp *qp = qptr;
 893         struct pending_queue *pend_q;
 894         struct cpt_request_info *req;
 895         struct rte_crypto_op *cop;
 896         uint8_t cc[nb_ops];
 897         struct rid *rid;
 898         uintptr_t *rsp;
 899         void *metabuf;
 900
 901         pend_q = &qp->pend_q;
 902
 903         nb_pending = pend_q->pending_count;
 904
 905         if (nb_ops > nb_pending)
 906                 nb_ops = nb_pending;
 907
 908         for (i = 0; i < nb_ops; i++) {
 909                 rid = &pend_q->rid_queue[pend_q->deq_head];
 910                 req = (struct cpt_request_info *)(rid->rid);
 911
 912                 cc[i] = otx2_cpt_compcode_get(req);
 913
 914                 if (unlikely(cc[i] == ERR_REQ_PENDING))
 915                         break;
 916
 917                 ops[i] = req->op;
 918
 919                 MOD_INC(pend_q->deq_head, OTX2_CPT_DEFAULT_CMD_QLEN);
 920                 pend_q->pending_count -= 1;
 921         }
 922
 923         nb_completed = i;
 924
 925         for (i = 0; i < nb_completed; i++) {
 926                 rsp = (void *)ops[i];
 927
 928                 metabuf = (void *)rsp[0];
 929                 cop = (void *)rsp[1];
 930
 931                 ops[i] = cop;
 932
 933                 otx2_cpt_dequeue_post_process(qp, cop, rsp, cc[i]);
 934
 935                 free_op_meta(metabuf, qp->meta_info.pool);
 936         }
 937
 938         return nb_completed;
 939 }
 940
 941 /* PMD ops */
 942
 943 static int
 944 otx2_cpt_dev_config(struct rte_cryptodev *dev,
 945                     struct rte_cryptodev_config *conf)
 946 {
 947         struct otx2_cpt_vf *vf = dev->data->dev_private;
 948         int ret;
 949
 950         if (conf->nb_queue_pairs > vf->max_queues) {
 951                 CPT_LOG_ERR("Invalid number of queue pairs requested");
 952                 return -EINVAL;
 953         }
 954
 955         dev->feature_flags &= ~conf->ff_disable;
 956
 957         if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {
 958                 /* Initialize shared FPM table */
 959                 ret = cpt_fpm_init(otx2_fpm_iova);
 960                 if (ret)
 961                         return ret;
 962         }
 963
 964         /* Unregister error interrupts */
 965         if (vf->err_intr_registered)
 966                 otx2_cpt_err_intr_unregister(dev);
 967
 968         /* Detach queues */
 969         if (vf->nb_queues) {
 970                 ret = otx2_cpt_queues_detach(dev);
 971                 if (ret) {
 972                         CPT_LOG_ERR("Could not detach CPT queues");
 973                         return ret;
 974                 }
 975         }
 976
 977         /* Attach queues */
 978         ret = otx2_cpt_queues_attach(dev, conf->nb_queue_pairs);
 979         if (ret) {
 980                 CPT_LOG_ERR("Could not attach CPT queues");
 981                 return -ENODEV;
 982         }
 983
 984         ret = otx2_cpt_msix_offsets_get(dev);
 985         if (ret) {
 986                 CPT_LOG_ERR("Could not get MSI-X offsets");
 987                 goto queues_detach;
 988         }
 989
 990         /* Register error interrupts */
 991         ret = otx2_cpt_err_intr_register(dev);
 992         if (ret) {
 993                 CPT_LOG_ERR("Could not register error interrupts");
 994                 goto queues_detach;
 995         }
 996
 997         ret = otx2_cpt_inline_init(dev);
 998         if (ret) {
 999                 CPT_LOG_ERR("Could not enable inline IPsec");
1000                 goto intr_unregister;
1001         }
1002
1003         dev->enqueue_burst = otx2_cpt_enqueue_burst;
1004         dev->dequeue_burst = otx2_cpt_dequeue_burst;
1005
1006         rte_mb();
1007         return 0;
1008
1009 intr_unregister:
1010         otx2_cpt_err_intr_unregister(dev);
1011 queues_detach:
1012         otx2_cpt_queues_detach(dev);
1013         return ret;
1014 }
1015
1016 static int
1017 otx2_cpt_dev_start(struct rte_cryptodev *dev)
1018 {
1019         RTE_SET_USED(dev);
1020
1021         CPT_PMD_INIT_FUNC_TRACE();
1022
1023         return 0;
1024 }
1025
1026 static void
1027 otx2_cpt_dev_stop(struct rte_cryptodev *dev)
1028 {
1029         CPT_PMD_INIT_FUNC_TRACE();
1030
1031         if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)
1032                 cpt_fpm_clear();
1033 }
1034
1035 static int
1036 otx2_cpt_dev_close(struct rte_cryptodev *dev)
1037 {
1038         struct otx2_cpt_vf *vf = dev->data->dev_private;
1039         int i, ret = 0;
1040
1041         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1042                 ret = otx2_cpt_queue_pair_release(dev, i);
1043                 if (ret)
1044                         return ret;
1045         }
1046
1047         /* Unregister error interrupts */
1048         if (vf->err_intr_registered)
1049                 otx2_cpt_err_intr_unregister(dev);
1050
1051         /* Detach queues */
1052         if (vf->nb_queues) {
1053                 ret = otx2_cpt_queues_detach(dev);
1054                 if (ret)
1055                         CPT_LOG_ERR("Could not detach CPT queues");
1056         }
1057
1058         return ret;
1059 }
1060
1061 static void
1062 otx2_cpt_dev_info_get(struct rte_cryptodev *dev,
1063                       struct rte_cryptodev_info *info)
1064 {
1065         struct otx2_cpt_vf *vf = dev->data->dev_private;
1066
1067         if (info != NULL) {
1068                 info->max_nb_queue_pairs = vf->max_queues;
1069                 info->feature_flags = dev->feature_flags;
1070                 info->capabilities = otx2_cpt_capabilities_get();
1071                 info->sym.max_nb_sessions = 0;
1072                 info->driver_id = otx2_cryptodev_driver_id;
1073                 info->min_mbuf_headroom_req = OTX2_CPT_MIN_HEADROOM_REQ;
1074                 info->min_mbuf_tailroom_req = OTX2_CPT_MIN_TAILROOM_REQ;
1075         }
1076 }
1077
1078 static int
1079 otx2_cpt_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1080                           const struct rte_cryptodev_qp_conf *conf,
1081                           int socket_id __rte_unused)
1082 {
1083         uint8_t grp_mask = OTX2_CPT_ENG_GRPS_MASK;
1084         struct rte_pci_device *pci_dev;
1085         struct otx2_cpt_qp *qp;
1086
1087         CPT_PMD_INIT_FUNC_TRACE();
1088
1089         if (dev->data->queue_pairs[qp_id] != NULL)
1090                 otx2_cpt_queue_pair_release(dev, qp_id);
1091
1092         if (conf->nb_descriptors > OTX2_CPT_DEFAULT_CMD_QLEN) {
1093                 CPT_LOG_ERR("Could not setup queue pair for %u descriptors",
1094                             conf->nb_descriptors);
1095                 return -EINVAL;
1096         }
1097
1098         pci_dev = RTE_DEV_TO_PCI(dev->device);
1099
1100         if (pci_dev->mem_resource[2].addr == NULL) {
1101                 CPT_LOG_ERR("Invalid PCI mem address");
1102                 return -EIO;
1103         }
1104
1105         qp = otx2_cpt_qp_create(dev, qp_id, grp_mask);
1106         if (qp == NULL) {
1107                 CPT_LOG_ERR("Could not create queue pair %d", qp_id);
1108                 return -ENOMEM;
1109         }
1110
1111         qp->sess_mp = conf->mp_session;
1112         qp->sess_mp_priv = conf->mp_session_private;
1113         dev->data->queue_pairs[qp_id] = qp;
1114
1115         return 0;
1116 }
1117
1118 static int
1119 otx2_cpt_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
1120 {
1121         struct otx2_cpt_qp *qp = dev->data->queue_pairs[qp_id];
1122         int ret;
1123
1124         CPT_PMD_INIT_FUNC_TRACE();
1125
1126         if (qp == NULL)
1127                 return -EINVAL;
1128
1129         CPT_LOG_INFO("Releasing queue pair %d", qp_id);
1130
1131         ret = otx2_cpt_qp_destroy(dev, qp);
1132         if (ret) {
1133                 CPT_LOG_ERR("Could not destroy queue pair %d", qp_id);
1134                 return ret;
1135         }
1136
1137         dev->data->queue_pairs[qp_id] = NULL;
1138
1139         return 0;
1140 }
1141
1142 static unsigned int
1143 otx2_cpt_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1144 {
1145         return cpt_get_session_size();
1146 }
1147
1148 static int
1149 otx2_cpt_sym_session_configure(struct rte_cryptodev *dev,
1150                                struct rte_crypto_sym_xform *xform,
1151                                struct rte_cryptodev_sym_session *sess,
1152                                struct rte_mempool *pool)
1153 {
1154         CPT_PMD_INIT_FUNC_TRACE();
1155
1156         return sym_session_configure(dev->driver_id, xform, sess, pool);
1157 }
1158
1159 static void
1160 otx2_cpt_sym_session_clear(struct rte_cryptodev *dev,
1161                            struct rte_cryptodev_sym_session *sess)
1162 {
1163         CPT_PMD_INIT_FUNC_TRACE();
1164
1165         return sym_session_clear(dev->driver_id, sess);
1166 }
1167
1168 static unsigned int
1169 otx2_cpt_asym_session_size_get(struct rte_cryptodev *dev __rte_unused)
1170 {
1171         return sizeof(struct cpt_asym_sess_misc);
1172 }
1173
1174 static int
1175 otx2_cpt_asym_session_cfg(struct rte_cryptodev *dev,
1176                           struct rte_crypto_asym_xform *xform,
1177                           struct rte_cryptodev_asym_session *sess,
1178                           struct rte_mempool *pool)
1179 {
1180         struct cpt_asym_sess_misc *priv;
1181         int ret;
1182
1183         CPT_PMD_INIT_FUNC_TRACE();
1184
1185         if (rte_mempool_get(pool, (void **)&priv)) {
1186                 CPT_LOG_ERR("Could not allocate session_private_data");
1187                 return -ENOMEM;
1188         }
1189
1190         memset(priv, 0, sizeof(struct cpt_asym_sess_misc));
1191
1192         ret = cpt_fill_asym_session_parameters(priv, xform);
1193         if (ret) {
1194                 CPT_LOG_ERR("Could not configure session parameters");
1195
1196                 /* Return session to mempool */
1197                 rte_mempool_put(pool, priv);
1198                 return ret;
1199         }
1200
1201         set_asym_session_private_data(sess, dev->driver_id, priv);
1202         return 0;
1203 }
1204
1205 static void
1206 otx2_cpt_asym_session_clear(struct rte_cryptodev *dev,
1207                             struct rte_cryptodev_asym_session *sess)
1208 {
1209         struct cpt_asym_sess_misc *priv;
1210         struct rte_mempool *sess_mp;
1211
1212         CPT_PMD_INIT_FUNC_TRACE();
1213
1214         priv = get_asym_session_private_data(sess, dev->driver_id);
1215         if (priv == NULL)
1216                 return;
1217
1218         /* Free resources allocated in session_cfg */
1219         cpt_free_asym_session_parameters(priv);
1220
1221         /* Reset and free object back to pool */
1222         memset(priv, 0, otx2_cpt_asym_session_size_get(dev));
1223         sess_mp = rte_mempool_from_obj(priv);
1224         set_asym_session_private_data(sess, dev->driver_id, NULL);
1225         rte_mempool_put(sess_mp, priv);
1226 }
1227
1228 struct rte_cryptodev_ops otx2_cpt_ops = {
1229         /* Device control ops */
1230         .dev_configure = otx2_cpt_dev_config,
1231         .dev_start = otx2_cpt_dev_start,
1232         .dev_stop = otx2_cpt_dev_stop,
1233         .dev_close = otx2_cpt_dev_close,
1234         .dev_infos_get = otx2_cpt_dev_info_get,
1235
1236         .stats_get = NULL,
1237         .stats_reset = NULL,
1238         .queue_pair_setup = otx2_cpt_queue_pair_setup,
1239         .queue_pair_release = otx2_cpt_queue_pair_release,
1240
1241         /* Symmetric crypto ops */
1242         .sym_session_get_size = otx2_cpt_sym_session_get_size,
1243         .sym_session_configure = otx2_cpt_sym_session_configure,
1244         .sym_session_clear = otx2_cpt_sym_session_clear,
1245
1246         /* Asymmetric crypto ops */
1247         .asym_session_get_size = otx2_cpt_asym_session_size_get,
1248         .asym_session_configure = otx2_cpt_asym_session_cfg,
1249         .asym_session_clear = otx2_cpt_asym_session_clear,
1250
1251 };