1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
8 #include <rte_common.h>
9 #include <rte_crypto_asym.h>
10 #include <rte_malloc.h>
13 #include "cnxk_cryptodev_ops.h"
16 enum rte_crypto_asym_xform_type xfrm_type;
18 struct rte_crypto_rsa_xform rsa_ctx;
19 struct rte_crypto_modex_xform mod_ctx;
20 struct roc_ae_ec_ctx ec_ctx;
22 uint64_t *cnxk_fpm_iova;
23 struct roc_ae_ec_group **ec_grp;
27 static __rte_always_inline void
28 cnxk_ae_modex_param_normalize(uint8_t **data, size_t *len)
32 /* Strip leading NUL bytes */
33 for (i = 0; i < *len; i++) {
41 static __rte_always_inline int
42 cnxk_ae_fill_modex_params(struct cnxk_ae_sess *sess,
43 struct rte_crypto_asym_xform *xform)
45 struct rte_crypto_modex_xform *ctx = &sess->mod_ctx;
46 size_t exp_len = xform->modex.exponent.length;
47 size_t mod_len = xform->modex.modulus.length;
48 uint8_t *exp = xform->modex.exponent.data;
49 uint8_t *mod = xform->modex.modulus.data;
51 cnxk_ae_modex_param_normalize(&mod, &mod_len);
52 cnxk_ae_modex_param_normalize(&exp, &exp_len);
54 if (unlikely(exp_len == 0 || mod_len == 0))
57 if (unlikely(exp_len > mod_len))
60 /* Allocate buffer to hold modexp params */
61 ctx->modulus.data = rte_malloc(NULL, mod_len + exp_len, 0);
62 if (ctx->modulus.data == NULL)
65 /* Set up modexp prime modulus and private exponent */
66 memcpy(ctx->modulus.data, mod, mod_len);
67 ctx->exponent.data = ctx->modulus.data + mod_len;
68 memcpy(ctx->exponent.data, exp, exp_len);
70 ctx->modulus.length = mod_len;
71 ctx->exponent.length = exp_len;
76 static __rte_always_inline int
77 cnxk_ae_fill_rsa_params(struct cnxk_ae_sess *sess,
78 struct rte_crypto_asym_xform *xform)
80 struct rte_crypto_rsa_priv_key_qt qt = xform->rsa.qt;
81 struct rte_crypto_rsa_xform *xfrm_rsa = &xform->rsa;
82 struct rte_crypto_rsa_xform *rsa = &sess->rsa_ctx;
83 size_t mod_len = xfrm_rsa->n.length;
84 size_t exp_len = xfrm_rsa->e.length;
85 size_t len = (mod_len / 2);
88 /* Make sure key length used is not more than mod_len/2 */
89 if (qt.p.data != NULL)
90 len = RTE_MIN(len, qt.p.length);
92 /* Total size required for RSA key params(n,e,(q,dQ,p,dP,qInv)) */
93 total_size = mod_len + exp_len + 5 * len;
95 /* Allocate buffer to hold all RSA keys */
96 rsa->n.data = rte_malloc(NULL, total_size, 0);
97 if (rsa->n.data == NULL)
100 /* Set up RSA prime modulus and public key exponent */
101 memcpy(rsa->n.data, xfrm_rsa->n.data, mod_len);
102 rsa->e.data = rsa->n.data + mod_len;
103 memcpy(rsa->e.data, xfrm_rsa->e.data, exp_len);
105 /* Private key in quintuple format */
107 rsa->qt.q.data = rsa->e.data + exp_len;
108 memcpy(rsa->qt.q.data, qt.q.data, qt.q.length);
109 rsa->qt.dQ.data = rsa->qt.q.data + qt.q.length;
110 memcpy(rsa->qt.dQ.data, qt.dQ.data, qt.dQ.length);
111 rsa->qt.p.data = rsa->qt.dQ.data + qt.dQ.length;
112 memcpy(rsa->qt.p.data, qt.p.data, qt.p.length);
113 rsa->qt.dP.data = rsa->qt.p.data + qt.p.length;
114 memcpy(rsa->qt.dP.data, qt.dP.data, qt.dP.length);
115 rsa->qt.qInv.data = rsa->qt.dP.data + qt.dP.length;
116 memcpy(rsa->qt.qInv.data, qt.qInv.data, qt.qInv.length);
118 rsa->qt.q.length = qt.q.length;
119 rsa->qt.dQ.length = qt.dQ.length;
120 rsa->qt.p.length = qt.p.length;
121 rsa->qt.dP.length = qt.dP.length;
122 rsa->qt.qInv.length = qt.qInv.length;
124 rsa->n.length = mod_len;
125 rsa->e.length = exp_len;
130 static __rte_always_inline int
131 cnxk_ae_fill_ec_params(struct cnxk_ae_sess *sess,
132 struct rte_crypto_asym_xform *xform)
134 struct roc_ae_ec_ctx *ec = &sess->ec_ctx;
136 switch (xform->ec.curve_id) {
137 case RTE_CRYPTO_EC_GROUP_SECP192R1:
138 ec->curveid = ROC_AE_EC_ID_P192;
140 case RTE_CRYPTO_EC_GROUP_SECP224R1:
141 ec->curveid = ROC_AE_EC_ID_P224;
143 case RTE_CRYPTO_EC_GROUP_SECP256R1:
144 ec->curveid = ROC_AE_EC_ID_P256;
146 case RTE_CRYPTO_EC_GROUP_SECP384R1:
147 ec->curveid = ROC_AE_EC_ID_P384;
149 case RTE_CRYPTO_EC_GROUP_SECP521R1:
150 ec->curveid = ROC_AE_EC_ID_P521;
153 /* Only NIST curves (FIPS 186-4) are supported */
160 static __rte_always_inline int
161 cnxk_ae_fill_session_parameters(struct cnxk_ae_sess *sess,
162 struct rte_crypto_asym_xform *xform)
166 sess->xfrm_type = xform->xform_type;
168 switch (xform->xform_type) {
169 case RTE_CRYPTO_ASYM_XFORM_RSA:
170 ret = cnxk_ae_fill_rsa_params(sess, xform);
172 case RTE_CRYPTO_ASYM_XFORM_MODEX:
173 ret = cnxk_ae_fill_modex_params(sess, xform);
175 case RTE_CRYPTO_ASYM_XFORM_ECDSA:
177 case RTE_CRYPTO_ASYM_XFORM_ECPM:
178 ret = cnxk_ae_fill_ec_params(sess, xform);
187 cnxk_ae_free_session_parameters(struct cnxk_ae_sess *sess)
189 struct rte_crypto_modex_xform *mod;
190 struct rte_crypto_rsa_xform *rsa;
192 switch (sess->xfrm_type) {
193 case RTE_CRYPTO_ASYM_XFORM_RSA:
194 rsa = &sess->rsa_ctx;
196 rte_free(rsa->n.data);
198 case RTE_CRYPTO_ASYM_XFORM_MODEX:
199 mod = &sess->mod_ctx;
200 if (mod->modulus.data)
201 rte_free(mod->modulus.data);
203 case RTE_CRYPTO_ASYM_XFORM_ECDSA:
205 case RTE_CRYPTO_ASYM_XFORM_ECPM:
212 static __rte_always_inline int
213 cnxk_ae_modex_prep(struct rte_crypto_op *op, struct roc_ae_buf_ptr *meta_buf,
214 struct rte_crypto_modex_xform *mod, struct cpt_inst_s *inst)
216 uint32_t exp_len = mod->exponent.length;
217 uint32_t mod_len = mod->modulus.length;
218 struct rte_crypto_mod_op_param mod_op;
219 uint64_t total_key_len;
220 union cpt_inst_w4 w4;
225 mod_op = op->asym->modex;
227 base_len = mod_op.base.length;
228 if (unlikely(base_len > mod_len)) {
229 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
233 total_key_len = mod_len + exp_len;
236 dptr = meta_buf->vaddr;
237 inst->dptr = (uintptr_t)dptr;
238 memcpy(dptr, mod->modulus.data, total_key_len);
239 dptr += total_key_len;
240 memcpy(dptr, mod_op.base.data, base_len);
242 dlen = total_key_len + base_len;
245 w4.s.opcode_major = ROC_AE_MAJOR_OP_MODEX;
246 w4.s.opcode_minor = ROC_AE_MINOR_OP_MODEX;
248 w4.s.param1 = mod_len;
249 w4.s.param2 = exp_len;
252 inst->w4.u64 = w4.u64;
253 inst->rptr = (uintptr_t)dptr;
258 static __rte_always_inline void
259 cnxk_ae_rsa_prep(struct rte_crypto_op *op, struct roc_ae_buf_ptr *meta_buf,
260 struct rte_crypto_rsa_xform *rsa,
261 rte_crypto_param *crypto_param, struct cpt_inst_s *inst)
263 struct rte_crypto_rsa_op_param rsa_op;
264 uint32_t mod_len = rsa->n.length;
265 uint32_t exp_len = rsa->e.length;
266 uint64_t total_key_len;
267 union cpt_inst_w4 w4;
272 rsa_op = op->asym->rsa;
273 total_key_len = mod_len + exp_len;
276 dptr = meta_buf->vaddr;
277 inst->dptr = (uintptr_t)dptr;
278 memcpy(dptr, rsa->n.data, total_key_len);
279 dptr += total_key_len;
281 in_size = crypto_param->length;
282 memcpy(dptr, crypto_param->data, in_size);
285 dlen = total_key_len + in_size;
287 if (rsa_op.pad == RTE_CRYPTO_RSA_PADDING_NONE) {
288 /* Use mod_exp operation for no_padding type */
289 w4.s.opcode_minor = ROC_AE_MINOR_OP_MODEX;
290 w4.s.param2 = exp_len;
292 if (rsa_op.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
293 w4.s.opcode_minor = ROC_AE_MINOR_OP_PKCS_ENC;
294 /* Public key encrypt, use BT2*/
295 w4.s.param2 = ROC_AE_CPT_BLOCK_TYPE2 |
296 ((uint16_t)(exp_len) << 1);
297 } else if (rsa_op.op_type == RTE_CRYPTO_ASYM_OP_VERIFY) {
298 w4.s.opcode_minor = ROC_AE_MINOR_OP_PKCS_DEC;
299 /* Public key decrypt, use BT1 */
300 w4.s.param2 = ROC_AE_CPT_BLOCK_TYPE1;
304 w4.s.opcode_major = ROC_AE_MAJOR_OP_MODEX;
306 w4.s.param1 = mod_len;
309 inst->w4.u64 = w4.u64;
310 inst->rptr = (uintptr_t)dptr;
313 static __rte_always_inline void
314 cnxk_ae_rsa_crt_prep(struct rte_crypto_op *op, struct roc_ae_buf_ptr *meta_buf,
315 struct rte_crypto_rsa_xform *rsa,
316 rte_crypto_param *crypto_param, struct cpt_inst_s *inst)
318 uint32_t qInv_len = rsa->qt.qInv.length;
319 struct rte_crypto_rsa_op_param rsa_op;
320 uint32_t dP_len = rsa->qt.dP.length;
321 uint32_t dQ_len = rsa->qt.dQ.length;
322 uint32_t p_len = rsa->qt.p.length;
323 uint32_t q_len = rsa->qt.q.length;
324 uint32_t mod_len = rsa->n.length;
325 uint64_t total_key_len;
326 union cpt_inst_w4 w4;
331 rsa_op = op->asym->rsa;
332 total_key_len = p_len + q_len + dP_len + dQ_len + qInv_len;
335 dptr = meta_buf->vaddr;
336 inst->dptr = (uintptr_t)dptr;
337 memcpy(dptr, rsa->qt.q.data, total_key_len);
338 dptr += total_key_len;
340 in_size = crypto_param->length;
341 memcpy(dptr, crypto_param->data, in_size);
344 dlen = total_key_len + in_size;
346 if (rsa_op.pad == RTE_CRYPTO_RSA_PADDING_NONE) {
347 /*Use mod_exp operation for no_padding type */
348 w4.s.opcode_minor = ROC_AE_MINOR_OP_MODEX_CRT;
350 if (rsa_op.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
351 w4.s.opcode_minor = ROC_AE_MINOR_OP_PKCS_ENC_CRT;
352 /* Private encrypt, use BT1 */
353 w4.s.param2 = ROC_AE_CPT_BLOCK_TYPE1;
354 } else if (rsa_op.op_type == RTE_CRYPTO_ASYM_OP_DECRYPT) {
355 w4.s.opcode_minor = ROC_AE_MINOR_OP_PKCS_DEC_CRT;
356 /* Private decrypt, use BT2 */
357 w4.s.param2 = ROC_AE_CPT_BLOCK_TYPE2;
361 w4.s.opcode_major = ROC_AE_MAJOR_OP_MODEX;
363 w4.s.param1 = mod_len;
366 inst->w4.u64 = w4.u64;
367 inst->rptr = (uintptr_t)dptr;
370 static __rte_always_inline int __rte_hot
371 cnxk_ae_enqueue_rsa_op(struct rte_crypto_op *op,
372 struct roc_ae_buf_ptr *meta_buf,
373 struct cnxk_ae_sess *sess, struct cpt_inst_s *inst)
375 struct rte_crypto_rsa_op_param *rsa = &op->asym->rsa;
377 switch (rsa->op_type) {
378 case RTE_CRYPTO_ASYM_OP_VERIFY:
379 cnxk_ae_rsa_prep(op, meta_buf, &sess->rsa_ctx, &rsa->sign,
382 case RTE_CRYPTO_ASYM_OP_ENCRYPT:
383 cnxk_ae_rsa_prep(op, meta_buf, &sess->rsa_ctx, &rsa->message,
386 case RTE_CRYPTO_ASYM_OP_SIGN:
387 cnxk_ae_rsa_crt_prep(op, meta_buf, &sess->rsa_ctx,
388 &rsa->message, inst);
390 case RTE_CRYPTO_ASYM_OP_DECRYPT:
391 cnxk_ae_rsa_crt_prep(op, meta_buf, &sess->rsa_ctx, &rsa->cipher,
395 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
401 static __rte_always_inline void
402 cnxk_ae_ecdsa_sign_prep(struct rte_crypto_ecdsa_op_param *ecdsa,
403 struct roc_ae_buf_ptr *meta_buf,
404 uint64_t fpm_table_iova, struct roc_ae_ec_group *ec_grp,
405 uint8_t curveid, struct cpt_inst_s *inst)
407 uint16_t message_len = ecdsa->message.length;
408 uint16_t pkey_len = ecdsa->pkey.length;
409 uint16_t p_align, k_align, m_align;
410 uint16_t k_len = ecdsa->k.length;
411 uint16_t order_len, prime_len;
412 uint16_t o_offset, pk_offset;
413 union cpt_inst_w4 w4;
417 prime_len = ec_grp->prime.length;
418 order_len = ec_grp->order.length;
420 /* Truncate input length to curve prime length */
421 if (message_len > prime_len)
422 message_len = prime_len;
423 m_align = RTE_ALIGN_CEIL(message_len, 8);
425 p_align = RTE_ALIGN_CEIL(prime_len, 8);
426 k_align = RTE_ALIGN_CEIL(k_len, 8);
428 /* Set write offset for order and private key */
429 o_offset = prime_len - order_len;
430 pk_offset = prime_len - pkey_len;
433 dptr = meta_buf->vaddr;
434 inst->dptr = (uintptr_t)dptr;
437 * Set dlen = sum(sizeof(fpm address), ROUNDUP8(scalar len, input len),
438 * ROUNDUP8(priv key len, prime len, order len)).
439 * Please note, private key, order cannot exceed prime
440 * length i.e 3 * p_align.
442 dlen = sizeof(fpm_table_iova) + k_align + m_align + p_align * 5;
444 memset(dptr, 0, dlen);
446 *(uint64_t *)dptr = fpm_table_iova;
447 dptr += sizeof(fpm_table_iova);
449 memcpy(dptr, ecdsa->k.data, k_len);
452 memcpy(dptr, ec_grp->prime.data, prime_len);
455 memcpy(dptr + o_offset, ec_grp->order.data, order_len);
458 memcpy(dptr + pk_offset, ecdsa->pkey.data, pkey_len);
461 memcpy(dptr, ecdsa->message.data, message_len);
464 memcpy(dptr, ec_grp->consta.data, prime_len);
467 memcpy(dptr, ec_grp->constb.data, prime_len);
471 w4.s.opcode_major = ROC_AE_MAJOR_OP_ECDSA;
472 w4.s.opcode_minor = ROC_AE_MINOR_OP_ECDSA_SIGN;
474 w4.s.param1 = curveid | (message_len << 8);
475 w4.s.param2 = (pkey_len << 8) | k_len;
478 inst->w4.u64 = w4.u64;
479 inst->rptr = (uintptr_t)dptr;
482 static __rte_always_inline void
483 cnxk_ae_ecdsa_verify_prep(struct rte_crypto_ecdsa_op_param *ecdsa,
484 struct roc_ae_buf_ptr *meta_buf,
485 uint64_t fpm_table_iova,
486 struct roc_ae_ec_group *ec_grp, uint8_t curveid,
487 struct cpt_inst_s *inst)
489 uint32_t message_len = ecdsa->message.length;
490 uint16_t o_offset, r_offset, s_offset;
491 uint16_t qx_len = ecdsa->q.x.length;
492 uint16_t qy_len = ecdsa->q.y.length;
493 uint16_t r_len = ecdsa->r.length;
494 uint16_t s_len = ecdsa->s.length;
495 uint16_t order_len, prime_len;
496 uint16_t qx_offset, qy_offset;
497 uint16_t p_align, m_align;
498 union cpt_inst_w4 w4;
502 prime_len = ec_grp->prime.length;
503 order_len = ec_grp->order.length;
505 /* Truncate input length to curve prime length */
506 if (message_len > prime_len)
507 message_len = prime_len;
509 m_align = RTE_ALIGN_CEIL(message_len, 8);
510 p_align = RTE_ALIGN_CEIL(prime_len, 8);
512 /* Set write offset for sign, order and public key coordinates */
513 o_offset = prime_len - order_len;
514 qx_offset = prime_len - qx_len;
515 qy_offset = prime_len - qy_len;
516 r_offset = prime_len - r_len;
517 s_offset = prime_len - s_len;
520 dptr = meta_buf->vaddr;
521 inst->dptr = (uintptr_t)dptr;
524 * Set dlen = sum(sizeof(fpm address), ROUNDUP8(message len),
525 * ROUNDUP8(sign len(r and s), public key len(x and y coordinates),
526 * prime len, order len)).
527 * Please note sign, public key and order can not exceed prime length
530 dlen = sizeof(fpm_table_iova) + m_align + (8 * p_align);
532 memset(dptr, 0, dlen);
534 *(uint64_t *)dptr = fpm_table_iova;
535 dptr += sizeof(fpm_table_iova);
537 memcpy(dptr + r_offset, ecdsa->r.data, r_len);
540 memcpy(dptr + s_offset, ecdsa->s.data, s_len);
543 memcpy(dptr, ecdsa->message.data, message_len);
546 memcpy(dptr + o_offset, ec_grp->order.data, order_len);
549 memcpy(dptr, ec_grp->prime.data, prime_len);
552 memcpy(dptr + qx_offset, ecdsa->q.x.data, qx_len);
555 memcpy(dptr + qy_offset, ecdsa->q.y.data, qy_len);
558 memcpy(dptr, ec_grp->consta.data, prime_len);
561 memcpy(dptr, ec_grp->constb.data, prime_len);
565 w4.s.opcode_major = ROC_AE_MAJOR_OP_ECDSA;
566 w4.s.opcode_minor = ROC_AE_MINOR_OP_ECDSA_VERIFY;
568 w4.s.param1 = curveid | (message_len << 8);
572 inst->w4.u64 = w4.u64;
573 inst->rptr = (uintptr_t)dptr;
576 static __rte_always_inline int __rte_hot
577 cnxk_ae_enqueue_ecdsa_op(struct rte_crypto_op *op,
578 struct roc_ae_buf_ptr *meta_buf,
579 struct cnxk_ae_sess *sess, uint64_t *fpm_iova,
580 struct roc_ae_ec_group **ec_grp,
581 struct cpt_inst_s *inst)
583 struct rte_crypto_ecdsa_op_param *ecdsa = &op->asym->ecdsa;
584 uint8_t curveid = sess->ec_ctx.curveid;
586 if (ecdsa->op_type == RTE_CRYPTO_ASYM_OP_SIGN)
587 cnxk_ae_ecdsa_sign_prep(ecdsa, meta_buf, fpm_iova[curveid],
588 ec_grp[curveid], curveid, inst);
589 else if (ecdsa->op_type == RTE_CRYPTO_ASYM_OP_VERIFY)
590 cnxk_ae_ecdsa_verify_prep(ecdsa, meta_buf, fpm_iova[curveid],
591 ec_grp[curveid], curveid, inst);
593 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
599 static __rte_always_inline int
600 cnxk_ae_ecpm_prep(struct rte_crypto_ecpm_op_param *ecpm,
601 struct roc_ae_buf_ptr *meta_buf,
602 struct roc_ae_ec_group *ec_grp, uint8_t curveid,
603 struct cpt_inst_s *inst)
605 uint16_t x1_len = ecpm->p.x.length;
606 uint16_t y1_len = ecpm->p.y.length;
607 uint16_t scalar_align, p_align;
608 uint16_t x1_offset, y1_offset;
609 uint16_t dlen, prime_len;
610 union cpt_inst_w4 w4;
613 prime_len = ec_grp->prime.length;
616 dptr = meta_buf->vaddr;
617 inst->dptr = (uintptr_t)dptr;
619 p_align = RTE_ALIGN_CEIL(prime_len, 8);
620 scalar_align = RTE_ALIGN_CEIL(ecpm->scalar.length, 8);
623 * Set dlen = sum(ROUNDUP8(input point(x and y coordinates), prime,
625 * Please note point length is equivalent to prime of the curve
627 dlen = 5 * p_align + scalar_align;
629 x1_offset = prime_len - x1_len;
630 y1_offset = prime_len - y1_len;
632 memset(dptr, 0, dlen);
634 /* Copy input point, scalar, prime */
635 memcpy(dptr + x1_offset, ecpm->p.x.data, x1_len);
637 memcpy(dptr + y1_offset, ecpm->p.y.data, y1_len);
639 memcpy(dptr, ecpm->scalar.data, ecpm->scalar.length);
640 dptr += scalar_align;
641 memcpy(dptr, ec_grp->prime.data, ec_grp->prime.length);
643 memcpy(dptr, ec_grp->consta.data, ec_grp->consta.length);
645 memcpy(dptr, ec_grp->constb.data, ec_grp->constb.length);
649 w4.s.opcode_major = ROC_AE_MAJOR_OP_ECC;
650 w4.s.opcode_minor = ROC_AE_MINOR_OP_ECC_UMP;
652 w4.s.param1 = curveid;
653 w4.s.param2 = ecpm->scalar.length;
656 inst->w4.u64 = w4.u64;
657 inst->rptr = (uintptr_t)dptr;
662 static __rte_always_inline void
663 cnxk_ae_dequeue_rsa_op(struct rte_crypto_op *cop, uint8_t *rptr,
664 struct rte_crypto_rsa_xform *rsa_ctx)
666 struct rte_crypto_rsa_op_param *rsa = &cop->asym->rsa;
668 switch (rsa->op_type) {
669 case RTE_CRYPTO_ASYM_OP_ENCRYPT:
670 rsa->cipher.length = rsa_ctx->n.length;
671 memcpy(rsa->cipher.data, rptr, rsa->cipher.length);
673 case RTE_CRYPTO_ASYM_OP_DECRYPT:
674 if (rsa->pad == RTE_CRYPTO_RSA_PADDING_NONE) {
675 rsa->message.length = rsa_ctx->n.length;
676 memcpy(rsa->message.data, rptr, rsa->message.length);
678 /* Get length of decrypted output */
679 rsa->message.length =
680 rte_cpu_to_be_16(*((uint16_t *)rptr));
682 * Offset output data pointer by length field
683 * (2 bytes) and copy decrypted data.
685 memcpy(rsa->message.data, rptr + 2,
686 rsa->message.length);
689 case RTE_CRYPTO_ASYM_OP_SIGN:
690 rsa->sign.length = rsa_ctx->n.length;
691 memcpy(rsa->sign.data, rptr, rsa->sign.length);
693 case RTE_CRYPTO_ASYM_OP_VERIFY:
694 if (rsa->pad == RTE_CRYPTO_RSA_PADDING_NONE) {
695 rsa->sign.length = rsa_ctx->n.length;
696 memcpy(rsa->sign.data, rptr, rsa->sign.length);
698 /* Get length of signed output */
700 rte_cpu_to_be_16(*((uint16_t *)rptr));
702 * Offset output data pointer by length field
703 * (2 bytes) and copy signed data.
705 memcpy(rsa->sign.data, rptr + 2, rsa->sign.length);
707 if (memcmp(rsa->sign.data, rsa->message.data,
708 rsa->message.length)) {
709 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
713 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
718 static __rte_always_inline void
719 cnxk_ae_dequeue_ecdsa_op(struct rte_crypto_ecdsa_op_param *ecdsa, uint8_t *rptr,
720 struct roc_ae_ec_ctx *ec,
721 struct roc_ae_ec_group **ec_grp)
723 int prime_len = ec_grp[ec->curveid]->prime.length;
725 if (ecdsa->op_type == RTE_CRYPTO_ASYM_OP_VERIFY)
728 /* Separate out sign r and s components */
729 memcpy(ecdsa->r.data, rptr, prime_len);
730 memcpy(ecdsa->s.data, rptr + RTE_ALIGN_CEIL(prime_len, 8), prime_len);
731 ecdsa->r.length = prime_len;
732 ecdsa->s.length = prime_len;
735 static __rte_always_inline void
736 cnxk_ae_dequeue_ecpm_op(struct rte_crypto_ecpm_op_param *ecpm, uint8_t *rptr,
737 struct roc_ae_ec_ctx *ec,
738 struct roc_ae_ec_group **ec_grp)
740 int prime_len = ec_grp[ec->curveid]->prime.length;
742 memcpy(ecpm->r.x.data, rptr, prime_len);
743 memcpy(ecpm->r.y.data, rptr + RTE_ALIGN_CEIL(prime_len, 8), prime_len);
744 ecpm->r.x.length = prime_len;
745 ecpm->r.y.length = prime_len;
748 static __rte_always_inline void *
749 cnxk_ae_alloc_meta(struct roc_ae_buf_ptr *buf,
750 struct rte_mempool *cpt_meta_pool,
751 struct cpt_inflight_req *infl_req)
755 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
760 infl_req->mdata = mdata;
761 infl_req->op_flags |= CPT_OP_FLAGS_METABUF;
766 static __rte_always_inline int32_t __rte_hot
767 cnxk_ae_enqueue(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
768 struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst,
769 struct cnxk_ae_sess *sess)
771 struct cpt_qp_meta_info *minfo = &qp->meta_info;
772 struct rte_crypto_asym_op *asym_op = op->asym;
773 struct roc_ae_buf_ptr meta_buf;
778 mdata = cnxk_ae_alloc_meta(&meta_buf, minfo->pool, infl_req);
782 /* Reserve 8B for RPTR */
783 meta_buf.vaddr = PLT_PTR_ADD(mdata, sizeof(uint64_t));
785 switch (sess->xfrm_type) {
786 case RTE_CRYPTO_ASYM_XFORM_MODEX:
787 ret = cnxk_ae_modex_prep(op, &meta_buf, &sess->mod_ctx, inst);
791 case RTE_CRYPTO_ASYM_XFORM_RSA:
792 ret = cnxk_ae_enqueue_rsa_op(op, &meta_buf, sess, inst);
796 case RTE_CRYPTO_ASYM_XFORM_ECDSA:
797 ret = cnxk_ae_enqueue_ecdsa_op(op, &meta_buf, sess,
803 case RTE_CRYPTO_ASYM_XFORM_ECPM:
804 ret = cnxk_ae_ecpm_prep(&asym_op->ecpm, &meta_buf,
805 sess->ec_grp[sess->ec_ctx.curveid],
806 sess->ec_ctx.curveid, inst);
811 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
821 rte_mempool_put(minfo->pool, infl_req->mdata);
825 static __rte_always_inline void
826 cnxk_ae_post_process(struct rte_crypto_op *cop, struct cnxk_ae_sess *sess,
829 struct rte_crypto_asym_op *op = cop->asym;
831 switch (sess->xfrm_type) {
832 case RTE_CRYPTO_ASYM_XFORM_RSA:
833 cnxk_ae_dequeue_rsa_op(cop, rptr, &sess->rsa_ctx);
835 case RTE_CRYPTO_ASYM_XFORM_MODEX:
836 op->modex.result.length = sess->mod_ctx.modulus.length;
837 memcpy(op->modex.result.data, rptr, op->modex.result.length);
839 case RTE_CRYPTO_ASYM_XFORM_ECDSA:
840 cnxk_ae_dequeue_ecdsa_op(&op->ecdsa, rptr, &sess->ec_ctx,
843 case RTE_CRYPTO_ASYM_XFORM_ECPM:
844 cnxk_ae_dequeue_ecpm_op(&op->ecpm, rptr, &sess->ec_ctx,
848 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
852 #endif /* _CNXK_AE_H_ */