1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 - 2022 Intel Corporation
7 #include <cryptodev_pmd.h>
9 #include "qat_device.h"
13 #include "icp_qat_fw_pke.h"
14 #include "icp_qat_fw.h"
18 uint8_t qat_asym_driver_id;
20 struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[QAT_N_GENS];
22 /* An rte_driver is needed in the registration of both the device and the driver
24 * The actual qat pci's rte_driver can't be used as its name represents
25 * the whole pci device with all services. Think of this as a holder for a name
26 * for the crypto part of the pci device.
28 static const char qat_asym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_ASYM_PMD);
29 static const struct rte_driver cryptodev_qat_asym_driver = {
30 .name = qat_asym_drv_name,
31 .alias = qat_asym_drv_name
35 * Macros with suffix _F are used with some of predefinded identifiers:
36 * - cookie->input_buffer
39 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
40 #define HEXDUMP(name, where, size) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
42 #define HEXDUMP_OFF(name, where, size, idx) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
43 &where[idx * size], size)
45 #define HEXDUMP_OFF_F(name, idx) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
46 &cookie->input_buffer[idx * qat_alg_bytesize], \
49 #define HEXDUMP(name, where, size)
50 #define HEXDUMP_OFF(name, where, size, idx)
51 #define HEXDUMP_OFF_F(name, idx)
54 #define CHECK_IF_NOT_EMPTY(param, name, pname, status) \
56 if (param.length == 0) { \
59 " input parameter, zero length " pname \
62 } else if (check_zero(param)) { \
64 "Invalid " name " input parameter, empty " \
65 pname ", length = %d", \
72 #define SET_PKE_LN(where, what, how, idx) \
73 rte_memcpy(where[idx] + how - \
78 #define SET_PKE_LN_9A(where, what, how, idx) \
79 rte_memcpy(&where[idx * RTE_ALIGN_CEIL(how, 8)] + \
80 RTE_ALIGN_CEIL(how, 8) - \
85 #define SET_PKE_LN_EC(where, what, how, idx) \
86 rte_memcpy(&where[idx * RTE_ALIGN_CEIL(how, 8)] + \
87 RTE_ALIGN_CEIL(how, 8) - \
92 #define SET_PKE_LN_9A_F(what, idx) \
93 rte_memcpy(&cookie->input_buffer[idx * qat_alg_bytesize] + \
94 qat_alg_bytesize - what.length, \
95 what.data, what.length)
97 #define SET_PKE_LN_EC_F(what, how, idx) \
98 rte_memcpy(&cookie->input_buffer[idx * \
99 RTE_ALIGN_CEIL(how, 8)] + \
100 RTE_ALIGN_CEIL(how, 8) - how, \
104 request_init(struct icp_qat_fw_pke_request *qat_req)
106 memset(qat_req, 0, sizeof(*qat_req));
107 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
108 qat_req->pke_hdr.hdr_flags =
109 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
110 (ICP_QAT_FW_COMN_REQ_FLAG_SET);
114 cleanup_arrays(struct qat_asym_op_cookie *cookie,
115 int in_count, int out_count, int alg_size)
119 for (i = 0; i < in_count; i++)
120 memset(cookie->input_array[i], 0x0, alg_size);
121 for (i = 0; i < out_count; i++)
122 memset(cookie->output_array[i], 0x0, alg_size);
126 cleanup_crt(struct qat_asym_op_cookie *cookie,
131 memset(cookie->input_array[0], 0x0, alg_size);
132 for (i = 1; i < QAT_ASYM_RSA_QT_NUM_IN_PARAMS; i++)
133 memset(cookie->input_array[i], 0x0, alg_size / 2);
134 for (i = 0; i < QAT_ASYM_RSA_NUM_OUT_PARAMS; i++)
135 memset(cookie->output_array[i], 0x0, alg_size);
139 cleanup(struct qat_asym_op_cookie *cookie,
140 struct rte_crypto_asym_xform *xform, int alg_size)
142 if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX)
143 cleanup_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
144 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, alg_size);
145 else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV)
146 cleanup_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
147 QAT_ASYM_MODINV_NUM_OUT_PARAMS, alg_size);
148 else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
149 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT)
150 cleanup_crt(cookie, alg_size);
152 cleanup_arrays(cookie, QAT_ASYM_RSA_NUM_IN_PARAMS,
153 QAT_ASYM_RSA_NUM_OUT_PARAMS, alg_size);
159 check_zero(rte_crypto_param n)
161 int i, len = n.length;
164 for (i = len - 1; i >= 0; i--) {
165 if (n.data[i] != 0x0)
168 } else if (len == 8 && *(uint64_t *)&n.data[len - 8] == 0) {
170 } else if (*(uint64_t *)&n.data[len - 8] == 0) {
171 for (i = len - 9; i >= 0; i--) {
172 if (n.data[i] != 0x0)
181 static struct qat_asym_function
182 get_asym_function(struct rte_crypto_asym_xform *xform)
184 struct qat_asym_function qat_function;
186 switch (xform->xform_type) {
187 case RTE_CRYPTO_ASYM_XFORM_MODEX:
188 qat_function = get_modexp_function(xform);
190 case RTE_CRYPTO_ASYM_XFORM_MODINV:
191 qat_function = get_modinv_function(xform);
194 qat_function.func_id = 0;
202 modexp_set_input(struct rte_crypto_asym_op *asym_op,
203 struct icp_qat_fw_pke_request *qat_req,
204 struct qat_asym_op_cookie *cookie,
205 struct rte_crypto_asym_xform *xform)
207 struct qat_asym_function qat_function;
208 uint32_t alg_bytesize, func_id;
211 CHECK_IF_NOT_EMPTY(xform->modex.modulus, "mod exp",
213 CHECK_IF_NOT_EMPTY(xform->modex.exponent, "mod exp",
218 qat_function = get_asym_function(xform);
219 func_id = qat_function.func_id;
220 if (qat_function.func_id == 0) {
221 QAT_LOG(ERR, "Cannot obtain functionality id");
224 alg_bytesize = qat_function.bytesize;
226 SET_PKE_LN(cookie->input_array, asym_op->modex.base,
228 SET_PKE_LN(cookie->input_array, xform->modex.exponent,
230 SET_PKE_LN(cookie->input_array, xform->modex.modulus,
233 cookie->alg_bytesize = alg_bytesize;
234 qat_req->pke_hdr.cd_pars.func_id = func_id;
235 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
236 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
238 HEXDUMP("ModExp base", cookie->input_array[0], alg_bytesize);
239 HEXDUMP("ModExp exponent", cookie->input_array[1], alg_bytesize);
240 HEXDUMP("ModExp modulus", cookie->input_array[2], alg_bytesize);
246 modexp_collect(struct rte_crypto_asym_op *asym_op,
247 struct qat_asym_op_cookie *cookie,
248 struct rte_crypto_asym_xform *xform)
250 rte_crypto_param n = xform->modex.modulus;
251 uint32_t alg_bytesize = cookie->alg_bytesize;
252 uint8_t *modexp_result = asym_op->modex.result.data;
254 rte_memcpy(modexp_result,
255 cookie->output_array[0] + alg_bytesize
256 - n.length, n.length);
257 HEXDUMP("ModExp result", cookie->output_array[0],
259 return RTE_CRYPTO_OP_STATUS_SUCCESS;
263 modinv_set_input(struct rte_crypto_asym_op *asym_op,
264 struct icp_qat_fw_pke_request *qat_req,
265 struct qat_asym_op_cookie *cookie,
266 struct rte_crypto_asym_xform *xform)
268 struct qat_asym_function qat_function;
269 uint32_t alg_bytesize, func_id;
272 CHECK_IF_NOT_EMPTY(xform->modex.modulus, "mod inv",
277 qat_function = get_asym_function(xform);
278 func_id = qat_function.func_id;
280 QAT_LOG(ERR, "Cannot obtain functionality id");
283 alg_bytesize = qat_function.bytesize;
285 SET_PKE_LN(cookie->input_array, asym_op->modinv.base,
287 SET_PKE_LN(cookie->input_array, xform->modinv.modulus,
290 cookie->alg_bytesize = alg_bytesize;
291 qat_req->pke_hdr.cd_pars.func_id = func_id;
292 qat_req->input_param_count =
293 QAT_ASYM_MODINV_NUM_IN_PARAMS;
294 qat_req->output_param_count =
295 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
297 HEXDUMP("ModInv base", cookie->input_array[0], alg_bytesize);
298 HEXDUMP("ModInv modulus", cookie->input_array[1], alg_bytesize);
304 modinv_collect(struct rte_crypto_asym_op *asym_op,
305 struct qat_asym_op_cookie *cookie,
306 struct rte_crypto_asym_xform *xform)
308 rte_crypto_param n = xform->modinv.modulus;
309 uint8_t *modinv_result = asym_op->modinv.result.data;
310 uint32_t alg_bytesize = cookie->alg_bytesize;
312 rte_memcpy(modinv_result + (asym_op->modinv.result.length
314 cookie->output_array[0] + alg_bytesize
315 - n.length, n.length);
316 HEXDUMP("ModInv result", cookie->output_array[0],
318 return RTE_CRYPTO_OP_STATUS_SUCCESS;
322 rsa_set_pub_input(struct rte_crypto_asym_op *asym_op,
323 struct icp_qat_fw_pke_request *qat_req,
324 struct qat_asym_op_cookie *cookie,
325 struct rte_crypto_asym_xform *xform)
327 struct qat_asym_function qat_function;
328 uint32_t alg_bytesize, func_id;
331 qat_function = get_rsa_enc_function(xform);
332 func_id = qat_function.func_id;
334 QAT_LOG(ERR, "Cannot obtain functionality id");
337 alg_bytesize = qat_function.bytesize;
339 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
340 switch (asym_op->rsa.pad) {
341 case RTE_CRYPTO_RSA_PADDING_NONE:
342 SET_PKE_LN(cookie->input_array, asym_op->rsa.message,
347 "Invalid RSA padding (Encryption)"
351 HEXDUMP("RSA Message", cookie->input_array[0], alg_bytesize);
353 switch (asym_op->rsa.pad) {
354 case RTE_CRYPTO_RSA_PADDING_NONE:
355 SET_PKE_LN(cookie->input_array, asym_op->rsa.sign,
360 "Invalid RSA padding (Verify)");
363 HEXDUMP("RSA Signature", cookie->input_array[0],
367 SET_PKE_LN(cookie->input_array, xform->rsa.e,
369 SET_PKE_LN(cookie->input_array, xform->rsa.n,
372 cookie->alg_bytesize = alg_bytesize;
373 qat_req->pke_hdr.cd_pars.func_id = func_id;
375 HEXDUMP("RSA Public Key", cookie->input_array[1], alg_bytesize);
376 HEXDUMP("RSA Modulus", cookie->input_array[2], alg_bytesize);
382 rsa_set_priv_input(struct rte_crypto_asym_op *asym_op,
383 struct icp_qat_fw_pke_request *qat_req,
384 struct qat_asym_op_cookie *cookie,
385 struct rte_crypto_asym_xform *xform)
387 struct qat_asym_function qat_function;
388 uint32_t alg_bytesize, func_id;
391 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT) {
392 qat_function = get_rsa_crt_function(xform);
393 func_id = qat_function.func_id;
395 QAT_LOG(ERR, "Cannot obtain functionality id");
398 alg_bytesize = qat_function.bytesize;
399 qat_req->input_param_count =
400 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
402 SET_PKE_LN(cookie->input_array, xform->rsa.qt.p,
403 (alg_bytesize >> 1), 1);
404 SET_PKE_LN(cookie->input_array, xform->rsa.qt.q,
405 (alg_bytesize >> 1), 2);
406 SET_PKE_LN(cookie->input_array, xform->rsa.qt.dP,
407 (alg_bytesize >> 1), 3);
408 SET_PKE_LN(cookie->input_array, xform->rsa.qt.dQ,
409 (alg_bytesize >> 1), 4);
410 SET_PKE_LN(cookie->input_array, xform->rsa.qt.qInv,
411 (alg_bytesize >> 1), 5);
413 HEXDUMP("RSA p", cookie->input_array[1],
415 HEXDUMP("RSA q", cookie->input_array[2],
417 HEXDUMP("RSA dP", cookie->input_array[3],
419 HEXDUMP("RSA dQ", cookie->input_array[4],
421 HEXDUMP("RSA qInv", cookie->input_array[5],
423 } else if (xform->rsa.key_type ==
424 RTE_RSA_KEY_TYPE_EXP) {
425 qat_function = get_rsa_dec_function(xform);
426 func_id = qat_function.func_id;
428 QAT_LOG(ERR, "Cannot obtain functionality id");
431 alg_bytesize = qat_function.bytesize;
433 SET_PKE_LN(cookie->input_array, xform->rsa.d,
435 SET_PKE_LN(cookie->input_array, xform->rsa.n,
438 HEXDUMP("RSA d", cookie->input_array[1],
440 HEXDUMP("RSA n", cookie->input_array[2],
443 QAT_LOG(ERR, "Invalid RSA key type");
447 if (asym_op->rsa.op_type ==
448 RTE_CRYPTO_ASYM_OP_DECRYPT) {
449 switch (asym_op->rsa.pad) {
450 case RTE_CRYPTO_RSA_PADDING_NONE:
451 SET_PKE_LN(cookie->input_array, asym_op->rsa.cipher,
453 HEXDUMP("RSA ciphertext", cookie->input_array[0],
458 "Invalid padding of RSA (Decrypt)");
462 } else if (asym_op->rsa.op_type ==
463 RTE_CRYPTO_ASYM_OP_SIGN) {
464 switch (asym_op->rsa.pad) {
465 case RTE_CRYPTO_RSA_PADDING_NONE:
466 SET_PKE_LN(cookie->input_array, asym_op->rsa.message,
468 HEXDUMP("RSA text to be signed", cookie->input_array[0],
473 "Invalid padding of RSA (Signature)");
478 cookie->alg_bytesize = alg_bytesize;
479 qat_req->pke_hdr.cd_pars.func_id = func_id;
484 rsa_set_input(struct rte_crypto_asym_op *asym_op,
485 struct icp_qat_fw_pke_request *qat_req,
486 struct qat_asym_op_cookie *cookie,
487 struct rte_crypto_asym_xform *xform)
489 qat_req->input_param_count =
490 QAT_ASYM_RSA_NUM_IN_PARAMS;
491 qat_req->output_param_count =
492 QAT_ASYM_RSA_NUM_OUT_PARAMS;
494 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
495 asym_op->rsa.op_type ==
496 RTE_CRYPTO_ASYM_OP_VERIFY) {
497 return rsa_set_pub_input(asym_op, qat_req, cookie, xform);
499 return rsa_set_priv_input(asym_op, qat_req, cookie, xform);
504 rsa_collect(struct rte_crypto_asym_op *asym_op,
505 struct qat_asym_op_cookie *cookie)
507 uint32_t alg_bytesize = cookie->alg_bytesize;
509 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
510 asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_VERIFY) {
512 if (asym_op->rsa.op_type ==
513 RTE_CRYPTO_ASYM_OP_ENCRYPT) {
514 uint8_t *rsa_result = asym_op->rsa.cipher.data;
516 rte_memcpy(rsa_result,
517 cookie->output_array[0],
519 HEXDUMP("RSA Encrypted data", cookie->output_array[0],
522 uint8_t *rsa_result = asym_op->rsa.cipher.data;
524 switch (asym_op->rsa.pad) {
525 case RTE_CRYPTO_RSA_PADDING_NONE:
526 rte_memcpy(rsa_result,
527 cookie->output_array[0],
529 HEXDUMP("RSA signature",
530 cookie->output_array[0],
534 QAT_LOG(ERR, "Padding not supported");
535 return RTE_CRYPTO_OP_STATUS_ERROR;
539 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_DECRYPT) {
540 uint8_t *rsa_result = asym_op->rsa.message.data;
542 switch (asym_op->rsa.pad) {
543 case RTE_CRYPTO_RSA_PADDING_NONE:
544 rte_memcpy(rsa_result,
545 cookie->output_array[0],
547 HEXDUMP("RSA Decrypted Message",
548 cookie->output_array[0],
552 QAT_LOG(ERR, "Padding not supported");
553 return RTE_CRYPTO_OP_STATUS_ERROR;
556 uint8_t *rsa_result = asym_op->rsa.sign.data;
558 rte_memcpy(rsa_result,
559 cookie->output_array[0],
561 HEXDUMP("RSA Signature", cookie->output_array[0],
565 return RTE_CRYPTO_OP_STATUS_SUCCESS;
569 ecdsa_set_input(struct rte_crypto_asym_op *asym_op,
570 struct icp_qat_fw_pke_request *qat_req,
571 struct qat_asym_op_cookie *cookie,
572 struct rte_crypto_asym_xform *xform)
574 struct qat_asym_function qat_function;
575 uint32_t alg_bytesize, qat_alg_bytesize, func_id;
578 curve_id = pick_curve(xform);
580 QAT_LOG(ERR, "Incorrect elliptic curve");
584 switch (asym_op->ecdsa.op_type) {
585 case RTE_CRYPTO_ASYM_OP_SIGN:
586 qat_function = get_ecdsa_function(xform);
587 func_id = qat_function.func_id;
589 QAT_LOG(ERR, "Cannot obtain functionality id");
592 alg_bytesize = qat_function.bytesize;
593 qat_alg_bytesize = RTE_ALIGN_CEIL(alg_bytesize, 8);
595 SET_PKE_LN_9A_F(asym_op->ecdsa.pkey, 0);
596 SET_PKE_LN_9A_F(asym_op->ecdsa.message, 1);
597 SET_PKE_LN_9A_F(asym_op->ecdsa.k, 2);
598 SET_PKE_LN_EC_F(curve[curve_id].b, alg_bytesize, 3);
599 SET_PKE_LN_EC_F(curve[curve_id].a, alg_bytesize, 4);
600 SET_PKE_LN_EC_F(curve[curve_id].p, alg_bytesize, 5);
601 SET_PKE_LN_EC_F(curve[curve_id].n, alg_bytesize, 6);
602 SET_PKE_LN_EC_F(curve[curve_id].y, alg_bytesize, 7);
603 SET_PKE_LN_EC_F(curve[curve_id].x, alg_bytesize, 8);
605 cookie->alg_bytesize = alg_bytesize;
606 qat_req->pke_hdr.cd_pars.func_id = func_id;
607 qat_req->input_param_count =
608 QAT_ASYM_ECDSA_RS_SIGN_IN_PARAMS;
609 qat_req->output_param_count =
610 QAT_ASYM_ECDSA_RS_SIGN_OUT_PARAMS;
612 HEXDUMP_OFF_F("ECDSA d", 0);
613 HEXDUMP_OFF_F("ECDSA e", 1);
614 HEXDUMP_OFF_F("ECDSA k", 2);
615 HEXDUMP_OFF_F("ECDSA b", 3);
616 HEXDUMP_OFF_F("ECDSA a", 4);
617 HEXDUMP_OFF_F("ECDSA n", 5);
618 HEXDUMP_OFF_F("ECDSA y", 6);
619 HEXDUMP_OFF_F("ECDSA x", 7);
621 case RTE_CRYPTO_ASYM_OP_VERIFY:
622 qat_function = get_ecdsa_verify_function(xform);
623 func_id = qat_function.func_id;
625 QAT_LOG(ERR, "Cannot obtain functionality id");
628 alg_bytesize = qat_function.bytesize;
629 qat_alg_bytesize = RTE_ALIGN_CEIL(alg_bytesize, 8);
631 SET_PKE_LN_9A_F(asym_op->ecdsa.message, 10);
632 SET_PKE_LN_9A_F(asym_op->ecdsa.s, 9);
633 SET_PKE_LN_9A_F(asym_op->ecdsa.r, 8);
634 SET_PKE_LN_EC_F(curve[curve_id].n, alg_bytesize, 7);
635 SET_PKE_LN_EC_F(curve[curve_id].x, alg_bytesize, 6);
636 SET_PKE_LN_EC_F(curve[curve_id].y, alg_bytesize, 5);
637 SET_PKE_LN_9A_F(asym_op->ecdsa.q.x, 4);
638 SET_PKE_LN_9A_F(asym_op->ecdsa.q.y, 3);
639 SET_PKE_LN_EC_F(curve[curve_id].a, alg_bytesize, 2);
640 SET_PKE_LN_EC_F(curve[curve_id].b, alg_bytesize, 1);
641 SET_PKE_LN_EC_F(curve[curve_id].p, alg_bytesize, 0);
643 cookie->alg_bytesize = alg_bytesize;
644 qat_req->pke_hdr.cd_pars.func_id = func_id;
645 qat_req->input_param_count =
646 QAT_ASYM_ECDSA_RS_VERIFY_IN_PARAMS;
647 qat_req->output_param_count =
648 QAT_ASYM_ECDSA_RS_VERIFY_OUT_PARAMS;
650 HEXDUMP_OFF_F("e", 0);
651 HEXDUMP_OFF_F("s", 1);
652 HEXDUMP_OFF_F("r", 2);
653 HEXDUMP_OFF_F("n", 3);
654 HEXDUMP_OFF_F("xG", 4);
655 HEXDUMP_OFF_F("yG", 5);
656 HEXDUMP_OFF_F("xQ", 6);
657 HEXDUMP_OFF_F("yQ", 7);
658 HEXDUMP_OFF_F("a", 8);
659 HEXDUMP_OFF_F("b", 9);
660 HEXDUMP_OFF_F("q", 10);
670 ecdsa_collect(struct rte_crypto_asym_op *asym_op,
671 struct qat_asym_op_cookie *cookie)
673 uint32_t alg_bytesize = RTE_ALIGN_CEIL(cookie->alg_bytesize, 8);
675 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
676 uint8_t *r = asym_op->ecdsa.r.data;
677 uint8_t *s = asym_op->ecdsa.s.data;
679 asym_op->ecdsa.r.length = alg_bytesize;
680 asym_op->ecdsa.s.length = alg_bytesize;
681 rte_memcpy(r, cookie->output_array[0], alg_bytesize);
682 rte_memcpy(s, cookie->output_array[1], alg_bytesize);
683 HEXDUMP("R", cookie->output_array[0],
685 HEXDUMP("S", cookie->output_array[1],
688 return RTE_CRYPTO_OP_STATUS_SUCCESS;
692 asym_set_input(struct rte_crypto_asym_op *asym_op,
693 struct icp_qat_fw_pke_request *qat_req,
694 struct qat_asym_op_cookie *cookie,
695 struct rte_crypto_asym_xform *xform)
697 switch (xform->xform_type) {
698 case RTE_CRYPTO_ASYM_XFORM_MODEX:
699 return modexp_set_input(asym_op, qat_req,
701 case RTE_CRYPTO_ASYM_XFORM_MODINV:
702 return modinv_set_input(asym_op, qat_req,
704 case RTE_CRYPTO_ASYM_XFORM_RSA:
705 return rsa_set_input(asym_op, qat_req,
707 case RTE_CRYPTO_ASYM_XFORM_ECDSA:
708 return ecdsa_set_input(asym_op, qat_req,
711 QAT_LOG(ERR, "Invalid/unsupported asymmetric crypto xform");
718 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
719 __rte_unused uint64_t *opaque,
720 __rte_unused enum qat_device_gen qat_dev_gen)
722 struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
723 struct rte_crypto_asym_op *asym_op = op->asym;
724 struct icp_qat_fw_pke_request *qat_req =
725 (struct icp_qat_fw_pke_request *)out_msg;
726 struct qat_asym_op_cookie *cookie =
727 (struct qat_asym_op_cookie *)op_cookie;
730 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
731 switch (op->sess_type) {
732 case RTE_CRYPTO_OP_WITH_SESSION:
734 "QAT asymmetric crypto PMD does not support session"
737 case RTE_CRYPTO_OP_SESSIONLESS:
738 request_init(qat_req);
739 err = asym_set_input(asym_op, qat_req, cookie,
742 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
747 QAT_DP_LOG(ERR, "Invalid session/xform settings");
748 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
752 qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
753 qat_req->pke_mid.src_data_addr = cookie->input_addr;
754 qat_req->pke_mid.dest_data_addr = cookie->output_addr;
756 HEXDUMP("qat_req:", qat_req, sizeof(struct icp_qat_fw_pke_request));
760 qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
761 HEXDUMP("qat_req:", qat_req, sizeof(struct icp_qat_fw_pke_request));
762 qat_req->output_param_count = 0;
763 qat_req->input_param_count = 0;
764 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
765 cookie->error |= err;
771 qat_asym_collect_response(struct rte_crypto_op *rx_op,
772 struct qat_asym_op_cookie *cookie,
773 struct rte_crypto_asym_xform *xform)
775 struct rte_crypto_asym_op *asym_op = rx_op->asym;
777 switch (xform->xform_type) {
778 case RTE_CRYPTO_ASYM_XFORM_MODEX:
779 return modexp_collect(asym_op, cookie, xform);
780 case RTE_CRYPTO_ASYM_XFORM_MODINV:
781 return modinv_collect(asym_op, cookie, xform);
782 case RTE_CRYPTO_ASYM_XFORM_RSA:
783 return rsa_collect(asym_op, cookie);
784 case RTE_CRYPTO_ASYM_XFORM_ECDSA:
785 return ecdsa_collect(asym_op, cookie);
787 QAT_LOG(ERR, "Not supported xform type");
788 return RTE_CRYPTO_OP_STATUS_ERROR;
793 qat_asym_process_response(void **op, uint8_t *resp,
794 void *op_cookie, __rte_unused uint64_t *dequeue_err_count)
796 struct icp_qat_fw_pke_resp *resp_msg =
797 (struct icp_qat_fw_pke_resp *)resp;
798 struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
800 struct qat_asym_op_cookie *cookie = op_cookie;
804 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
805 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
806 QAT_DP_LOG(ERR, "Cookie status returned error");
808 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
809 resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
810 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
811 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
812 QAT_DP_LOG(ERR, "Asymmetric response status"
815 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
816 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
817 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
818 QAT_DP_LOG(ERR, "Asymmetric common status"
822 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
823 rx_op->status = qat_asym_collect_response(rx_op,
824 cookie, rx_op->asym->xform);
825 cleanup(cookie, rx_op->asym->xform,
826 cookie->alg_bytesize);
830 HEXDUMP("resp_msg:", resp_msg, sizeof(struct icp_qat_fw_pke_resp));
836 qat_asym_session_configure(struct rte_cryptodev *dev __rte_unused,
837 struct rte_crypto_asym_xform *xform __rte_unused,
838 struct rte_cryptodev_asym_session *sess __rte_unused)
840 QAT_LOG(ERR, "QAT asymmetric PMD currently does not support session");
845 qat_asym_session_get_private_size(struct rte_cryptodev *dev __rte_unused)
847 QAT_LOG(ERR, "QAT asymmetric PMD currently does not support session");
852 qat_asym_session_clear(struct rte_cryptodev *dev __rte_unused,
853 struct rte_cryptodev_asym_session *sess __rte_unused)
855 QAT_LOG(ERR, "QAT asymmetric PMD currently does not support session");
859 qat_asym_crypto_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
862 return qat_enqueue_op_burst(qp, qat_asym_build_request, (void **)ops,
867 qat_asym_crypto_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
870 return qat_dequeue_op_burst(qp, (void **)ops, qat_asym_process_response,
875 qat_asym_init_op_cookie(void *op_cookie)
878 struct qat_asym_op_cookie *cookie = op_cookie;
880 cookie->input_addr = rte_mempool_virt2iova(cookie) +
881 offsetof(struct qat_asym_op_cookie,
884 cookie->output_addr = rte_mempool_virt2iova(cookie) +
885 offsetof(struct qat_asym_op_cookie,
888 for (j = 0; j < 8; j++) {
889 cookie->input_params_ptrs[j] =
890 rte_mempool_virt2iova(cookie) +
891 offsetof(struct qat_asym_op_cookie,
893 cookie->output_params_ptrs[j] =
894 rte_mempool_virt2iova(cookie) +
895 offsetof(struct qat_asym_op_cookie,
901 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
902 struct qat_dev_cmd_param *qat_dev_cmd_param)
904 struct qat_cryptodev_private *internals;
905 struct rte_cryptodev *cryptodev;
906 struct qat_device_info *qat_dev_instance =
907 &qat_pci_devs[qat_pci_dev->qat_dev_id];
908 struct rte_cryptodev_pmd_init_params init_params = {
910 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
911 .private_data_size = sizeof(struct qat_cryptodev_private)
913 struct qat_capabilities_info capa_info;
914 const struct rte_cryptodev_capabilities *capabilities;
915 const struct qat_crypto_gen_dev_ops *gen_dev_ops =
916 &qat_asym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
917 char name[RTE_CRYPTODEV_NAME_MAX_LEN];
918 char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
922 snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
923 qat_pci_dev->name, "asym");
924 QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name);
926 if (gen_dev_ops->cryptodev_ops == NULL) {
927 QAT_LOG(ERR, "Device %s does not support asymmetric crypto",
932 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
933 qat_pci_dev->qat_asym_driver_id =
935 } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
936 if (qat_pci_dev->qat_asym_driver_id !=
937 qat_asym_driver_id) {
939 "Device %s have different driver id than corresponding device in primary process",
945 /* Populate subset device to use in cryptodev device creation */
946 qat_dev_instance->asym_rte_dev.driver = &cryptodev_qat_asym_driver;
947 qat_dev_instance->asym_rte_dev.numa_node =
948 qat_dev_instance->pci_dev->device.numa_node;
949 qat_dev_instance->asym_rte_dev.devargs = NULL;
951 cryptodev = rte_cryptodev_pmd_create(name,
952 &(qat_dev_instance->asym_rte_dev), &init_params);
954 if (cryptodev == NULL)
957 qat_dev_instance->asym_rte_dev.name = cryptodev->data->name;
958 cryptodev->driver_id = qat_asym_driver_id;
959 cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
961 cryptodev->enqueue_burst = qat_asym_crypto_enqueue_op_burst;
962 cryptodev->dequeue_burst = qat_asym_crypto_dequeue_op_burst;
964 cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
966 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
969 snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
970 "QAT_ASYM_CAPA_GEN_%d",
971 qat_pci_dev->qat_dev_gen);
973 internals = cryptodev->data->dev_private;
974 internals->qat_dev = qat_pci_dev;
975 internals->dev_id = cryptodev->data->dev_id;
977 capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
978 capabilities = capa_info.data;
979 capa_size = capa_info.size;
981 internals->capa_mz = rte_memzone_lookup(capa_memz_name);
982 if (internals->capa_mz == NULL) {
983 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
984 capa_size, rte_socket_id(), 0);
985 if (internals->capa_mz == NULL) {
987 "Error allocating memzone for capabilities, "
988 "destroying PMD for %s",
990 rte_cryptodev_pmd_destroy(cryptodev);
991 memset(&qat_dev_instance->asym_rte_dev, 0,
992 sizeof(qat_dev_instance->asym_rte_dev));
997 memcpy(internals->capa_mz->addr, capabilities, capa_size);
998 internals->qat_dev_capabilities = internals->capa_mz->addr;
1001 if (qat_dev_cmd_param[i].name == NULL)
1003 if (!strcmp(qat_dev_cmd_param[i].name, ASYM_ENQ_THRESHOLD_NAME))
1004 internals->min_enq_burst_threshold =
1005 qat_dev_cmd_param[i].val;
1009 qat_pci_dev->asym_dev = internals;
1010 internals->service_type = QAT_SERVICE_ASYMMETRIC;
1011 QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
1012 cryptodev->data->name, internals->dev_id);
1017 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
1019 struct rte_cryptodev *cryptodev;
1021 if (qat_pci_dev == NULL)
1023 if (qat_pci_dev->asym_dev == NULL)
1025 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
1026 rte_memzone_free(qat_pci_dev->asym_dev->capa_mz);
1028 /* free crypto device */
1029 cryptodev = rte_cryptodev_pmd_get_dev(
1030 qat_pci_dev->asym_dev->dev_id);
1031 rte_cryptodev_pmd_destroy(cryptodev);
1032 qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
1033 qat_pci_dev->asym_dev = NULL;
1038 static struct cryptodev_driver qat_crypto_drv;
1039 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
1040 cryptodev_qat_asym_driver,
1041 qat_asym_driver_id);