1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 - 2022 Intel Corporation
7 #include <cryptodev_pmd.h>
9 #include "qat_device.h"
13 #include "icp_qat_fw_pke.h"
14 #include "icp_qat_fw.h"
18 uint8_t qat_asym_driver_id;
20 struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[QAT_N_GENS];
22 /* An rte_driver is needed in the registration of both the device and the driver
24 * The actual qat pci's rte_driver can't be used as its name represents
25 * the whole pci device with all services. Think of this as a holder for a name
26 * for the crypto part of the pci device.
28 static const char qat_asym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_ASYM_PMD);
29 static const struct rte_driver cryptodev_qat_asym_driver = {
30 .name = qat_asym_drv_name,
31 .alias = qat_asym_drv_name
34 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
35 #define HEXDUMP(name, where, size) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
37 #define HEXDUMP_OFF(name, where, size, idx) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
38 &where[idx * size], size)
40 #define HEXDUMP(name, where, size)
41 #define HEXDUMP_OFF(name, where, size, idx)
44 #define CHECK_IF_NOT_EMPTY(param, name, pname, status) \
46 if (param.length == 0) { \
49 " input parameter, zero length " pname \
52 } else if (check_zero(param)) { \
54 "Invalid " name " input parameter, empty " \
55 pname ", length = %d", \
62 #define SET_PKE_LN(where, what, how, idx) \
63 rte_memcpy(where[idx] + how - \
68 #define SET_PKE_LN_9A(where, what, how, idx) \
69 rte_memcpy(&where[idx * RTE_ALIGN_CEIL(how, 8)] + \
70 RTE_ALIGN_CEIL(how, 8) - \
75 #define SET_PKE_LN_EC(where, what, how, idx) \
76 rte_memcpy(&where[idx * RTE_ALIGN_CEIL(how, 8)] + \
77 RTE_ALIGN_CEIL(how, 8) - \
83 request_init(struct icp_qat_fw_pke_request *qat_req)
85 memset(qat_req, 0, sizeof(*qat_req));
86 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
87 qat_req->pke_hdr.hdr_flags =
88 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
89 (ICP_QAT_FW_COMN_REQ_FLAG_SET);
93 cleanup_arrays(struct qat_asym_op_cookie *cookie,
94 int in_count, int out_count, int alg_size)
98 for (i = 0; i < in_count; i++)
99 memset(cookie->input_array[i], 0x0, alg_size);
100 for (i = 0; i < out_count; i++)
101 memset(cookie->output_array[i], 0x0, alg_size);
105 cleanup_crt(struct qat_asym_op_cookie *cookie,
110 memset(cookie->input_array[0], 0x0, alg_size);
111 for (i = 1; i < QAT_ASYM_RSA_QT_NUM_IN_PARAMS; i++)
112 memset(cookie->input_array[i], 0x0, alg_size / 2);
113 for (i = 0; i < QAT_ASYM_RSA_NUM_OUT_PARAMS; i++)
114 memset(cookie->output_array[i], 0x0, alg_size);
118 cleanup(struct qat_asym_op_cookie *cookie,
119 struct rte_crypto_asym_xform *xform, int alg_size)
121 if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX)
122 cleanup_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
123 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, alg_size);
124 else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV)
125 cleanup_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
126 QAT_ASYM_MODINV_NUM_OUT_PARAMS, alg_size);
127 else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
128 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT)
129 cleanup_crt(cookie, alg_size);
131 cleanup_arrays(cookie, QAT_ASYM_RSA_NUM_IN_PARAMS,
132 QAT_ASYM_RSA_NUM_OUT_PARAMS, alg_size);
138 check_zero(rte_crypto_param n)
140 int i, len = n.length;
143 for (i = len - 1; i >= 0; i--) {
144 if (n.data[i] != 0x0)
147 } else if (len == 8 && *(uint64_t *)&n.data[len - 8] == 0) {
149 } else if (*(uint64_t *)&n.data[len - 8] == 0) {
150 for (i = len - 9; i >= 0; i--) {
151 if (n.data[i] != 0x0)
160 static struct qat_asym_function
161 get_asym_function(struct rte_crypto_asym_xform *xform)
163 struct qat_asym_function qat_function;
165 switch (xform->xform_type) {
166 case RTE_CRYPTO_ASYM_XFORM_MODEX:
167 qat_function = get_modexp_function(xform);
169 case RTE_CRYPTO_ASYM_XFORM_MODINV:
170 qat_function = get_modinv_function(xform);
173 qat_function.func_id = 0;
181 modexp_set_input(struct rte_crypto_asym_op *asym_op,
182 struct icp_qat_fw_pke_request *qat_req,
183 struct qat_asym_op_cookie *cookie,
184 struct rte_crypto_asym_xform *xform)
186 struct qat_asym_function qat_function;
187 uint32_t alg_bytesize, func_id;
190 CHECK_IF_NOT_EMPTY(xform->modex.modulus, "mod exp",
192 CHECK_IF_NOT_EMPTY(xform->modex.exponent, "mod exp",
197 qat_function = get_asym_function(xform);
198 func_id = qat_function.func_id;
199 if (qat_function.func_id == 0) {
200 QAT_LOG(ERR, "Cannot obtain functionality id");
203 alg_bytesize = qat_function.bytesize;
205 SET_PKE_LN(cookie->input_array, asym_op->modex.base,
207 SET_PKE_LN(cookie->input_array, xform->modex.exponent,
209 SET_PKE_LN(cookie->input_array, xform->modex.modulus,
212 cookie->alg_bytesize = alg_bytesize;
213 qat_req->pke_hdr.cd_pars.func_id = func_id;
214 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
215 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
217 HEXDUMP("ModExp base", cookie->input_array[0], alg_bytesize);
218 HEXDUMP("ModExp exponent", cookie->input_array[1], alg_bytesize);
219 HEXDUMP("ModExp modulus", cookie->input_array[2], alg_bytesize);
225 modexp_collect(struct rte_crypto_asym_op *asym_op,
226 struct qat_asym_op_cookie *cookie,
227 struct rte_crypto_asym_xform *xform)
229 rte_crypto_param n = xform->modex.modulus;
230 uint32_t alg_bytesize = cookie->alg_bytesize;
231 uint8_t *modexp_result = asym_op->modex.result.data;
233 rte_memcpy(modexp_result,
234 cookie->output_array[0] + alg_bytesize
235 - n.length, n.length);
236 HEXDUMP("ModExp result", cookie->output_array[0],
238 return RTE_CRYPTO_OP_STATUS_SUCCESS;
242 modinv_set_input(struct rte_crypto_asym_op *asym_op,
243 struct icp_qat_fw_pke_request *qat_req,
244 struct qat_asym_op_cookie *cookie,
245 struct rte_crypto_asym_xform *xform)
247 struct qat_asym_function qat_function;
248 uint32_t alg_bytesize, func_id;
251 CHECK_IF_NOT_EMPTY(xform->modex.modulus, "mod inv",
256 qat_function = get_asym_function(xform);
257 func_id = qat_function.func_id;
259 QAT_LOG(ERR, "Cannot obtain functionality id");
262 alg_bytesize = qat_function.bytesize;
264 SET_PKE_LN(cookie->input_array, asym_op->modinv.base,
266 SET_PKE_LN(cookie->input_array, xform->modinv.modulus,
269 cookie->alg_bytesize = alg_bytesize;
270 qat_req->pke_hdr.cd_pars.func_id = func_id;
271 qat_req->input_param_count =
272 QAT_ASYM_MODINV_NUM_IN_PARAMS;
273 qat_req->output_param_count =
274 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
276 HEXDUMP("ModInv base", cookie->input_array[0], alg_bytesize);
277 HEXDUMP("ModInv modulus", cookie->input_array[1], alg_bytesize);
283 modinv_collect(struct rte_crypto_asym_op *asym_op,
284 struct qat_asym_op_cookie *cookie,
285 struct rte_crypto_asym_xform *xform)
287 rte_crypto_param n = xform->modinv.modulus;
288 uint8_t *modinv_result = asym_op->modinv.result.data;
289 uint32_t alg_bytesize = cookie->alg_bytesize;
291 rte_memcpy(modinv_result + (asym_op->modinv.result.length
293 cookie->output_array[0] + alg_bytesize
294 - n.length, n.length);
295 HEXDUMP("ModInv result", cookie->output_array[0],
297 return RTE_CRYPTO_OP_STATUS_SUCCESS;
301 rsa_set_pub_input(struct rte_crypto_asym_op *asym_op,
302 struct icp_qat_fw_pke_request *qat_req,
303 struct qat_asym_op_cookie *cookie,
304 struct rte_crypto_asym_xform *xform)
306 struct qat_asym_function qat_function;
307 uint32_t alg_bytesize, func_id;
310 qat_function = get_rsa_enc_function(xform);
311 func_id = qat_function.func_id;
313 QAT_LOG(ERR, "Cannot obtain functionality id");
316 alg_bytesize = qat_function.bytesize;
318 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
319 switch (asym_op->rsa.pad) {
320 case RTE_CRYPTO_RSA_PADDING_NONE:
321 SET_PKE_LN(cookie->input_array, asym_op->rsa.message,
326 "Invalid RSA padding (Encryption)"
330 HEXDUMP("RSA Message", cookie->input_array[0], alg_bytesize);
332 switch (asym_op->rsa.pad) {
333 case RTE_CRYPTO_RSA_PADDING_NONE:
334 SET_PKE_LN(cookie->input_array, asym_op->rsa.sign,
339 "Invalid RSA padding (Verify)");
342 HEXDUMP("RSA Signature", cookie->input_array[0],
346 SET_PKE_LN(cookie->input_array, xform->rsa.e,
348 SET_PKE_LN(cookie->input_array, xform->rsa.n,
351 cookie->alg_bytesize = alg_bytesize;
352 qat_req->pke_hdr.cd_pars.func_id = func_id;
354 HEXDUMP("RSA Public Key", cookie->input_array[1], alg_bytesize);
355 HEXDUMP("RSA Modulus", cookie->input_array[2], alg_bytesize);
361 rsa_set_priv_input(struct rte_crypto_asym_op *asym_op,
362 struct icp_qat_fw_pke_request *qat_req,
363 struct qat_asym_op_cookie *cookie,
364 struct rte_crypto_asym_xform *xform)
366 struct qat_asym_function qat_function;
367 uint32_t alg_bytesize, func_id;
370 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT) {
371 qat_function = get_rsa_crt_function(xform);
372 func_id = qat_function.func_id;
374 QAT_LOG(ERR, "Cannot obtain functionality id");
377 alg_bytesize = qat_function.bytesize;
378 qat_req->input_param_count =
379 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
381 SET_PKE_LN(cookie->input_array, xform->rsa.qt.p,
382 (alg_bytesize >> 1), 1);
383 SET_PKE_LN(cookie->input_array, xform->rsa.qt.q,
384 (alg_bytesize >> 1), 2);
385 SET_PKE_LN(cookie->input_array, xform->rsa.qt.dP,
386 (alg_bytesize >> 1), 3);
387 SET_PKE_LN(cookie->input_array, xform->rsa.qt.dQ,
388 (alg_bytesize >> 1), 4);
389 SET_PKE_LN(cookie->input_array, xform->rsa.qt.qInv,
390 (alg_bytesize >> 1), 5);
392 HEXDUMP("RSA p", cookie->input_array[1],
394 HEXDUMP("RSA q", cookie->input_array[2],
396 HEXDUMP("RSA dP", cookie->input_array[3],
398 HEXDUMP("RSA dQ", cookie->input_array[4],
400 HEXDUMP("RSA qInv", cookie->input_array[5],
402 } else if (xform->rsa.key_type ==
403 RTE_RSA_KEY_TYPE_EXP) {
404 qat_function = get_rsa_dec_function(xform);
405 func_id = qat_function.func_id;
407 QAT_LOG(ERR, "Cannot obtain functionality id");
410 alg_bytesize = qat_function.bytesize;
412 SET_PKE_LN(cookie->input_array, xform->rsa.d,
414 SET_PKE_LN(cookie->input_array, xform->rsa.n,
417 HEXDUMP("RSA d", cookie->input_array[1],
419 HEXDUMP("RSA n", cookie->input_array[2],
422 QAT_LOG(ERR, "Invalid RSA key type");
426 if (asym_op->rsa.op_type ==
427 RTE_CRYPTO_ASYM_OP_DECRYPT) {
428 switch (asym_op->rsa.pad) {
429 case RTE_CRYPTO_RSA_PADDING_NONE:
430 SET_PKE_LN(cookie->input_array, asym_op->rsa.cipher,
432 HEXDUMP("RSA ciphertext", cookie->input_array[0],
437 "Invalid padding of RSA (Decrypt)");
441 } else if (asym_op->rsa.op_type ==
442 RTE_CRYPTO_ASYM_OP_SIGN) {
443 switch (asym_op->rsa.pad) {
444 case RTE_CRYPTO_RSA_PADDING_NONE:
445 SET_PKE_LN(cookie->input_array, asym_op->rsa.message,
447 HEXDUMP("RSA text to be signed", cookie->input_array[0],
452 "Invalid padding of RSA (Signature)");
457 cookie->alg_bytesize = alg_bytesize;
458 qat_req->pke_hdr.cd_pars.func_id = func_id;
463 rsa_set_input(struct rte_crypto_asym_op *asym_op,
464 struct icp_qat_fw_pke_request *qat_req,
465 struct qat_asym_op_cookie *cookie,
466 struct rte_crypto_asym_xform *xform)
468 qat_req->input_param_count =
469 QAT_ASYM_RSA_NUM_IN_PARAMS;
470 qat_req->output_param_count =
471 QAT_ASYM_RSA_NUM_OUT_PARAMS;
473 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
474 asym_op->rsa.op_type ==
475 RTE_CRYPTO_ASYM_OP_VERIFY) {
476 return rsa_set_pub_input(asym_op, qat_req, cookie, xform);
478 return rsa_set_priv_input(asym_op, qat_req, cookie, xform);
483 rsa_collect(struct rte_crypto_asym_op *asym_op,
484 struct qat_asym_op_cookie *cookie)
486 uint32_t alg_bytesize = cookie->alg_bytesize;
488 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
489 asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_VERIFY) {
491 if (asym_op->rsa.op_type ==
492 RTE_CRYPTO_ASYM_OP_ENCRYPT) {
493 uint8_t *rsa_result = asym_op->rsa.cipher.data;
495 rte_memcpy(rsa_result,
496 cookie->output_array[0],
498 HEXDUMP("RSA Encrypted data", cookie->output_array[0],
501 uint8_t *rsa_result = asym_op->rsa.cipher.data;
503 switch (asym_op->rsa.pad) {
504 case RTE_CRYPTO_RSA_PADDING_NONE:
505 rte_memcpy(rsa_result,
506 cookie->output_array[0],
508 HEXDUMP("RSA signature",
509 cookie->output_array[0],
513 QAT_LOG(ERR, "Padding not supported");
514 return RTE_CRYPTO_OP_STATUS_ERROR;
518 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_DECRYPT) {
519 uint8_t *rsa_result = asym_op->rsa.message.data;
521 switch (asym_op->rsa.pad) {
522 case RTE_CRYPTO_RSA_PADDING_NONE:
523 rte_memcpy(rsa_result,
524 cookie->output_array[0],
526 HEXDUMP("RSA Decrypted Message",
527 cookie->output_array[0],
531 QAT_LOG(ERR, "Padding not supported");
532 return RTE_CRYPTO_OP_STATUS_ERROR;
535 uint8_t *rsa_result = asym_op->rsa.sign.data;
537 rte_memcpy(rsa_result,
538 cookie->output_array[0],
540 HEXDUMP("RSA Signature", cookie->output_array[0],
544 return RTE_CRYPTO_OP_STATUS_SUCCESS;
549 asym_set_input(struct rte_crypto_asym_op *asym_op,
550 struct icp_qat_fw_pke_request *qat_req,
551 struct qat_asym_op_cookie *cookie,
552 struct rte_crypto_asym_xform *xform)
554 switch (xform->xform_type) {
555 case RTE_CRYPTO_ASYM_XFORM_MODEX:
556 return modexp_set_input(asym_op, qat_req,
558 case RTE_CRYPTO_ASYM_XFORM_MODINV:
559 return modinv_set_input(asym_op, qat_req,
561 case RTE_CRYPTO_ASYM_XFORM_RSA:
562 return rsa_set_input(asym_op, qat_req,
565 QAT_LOG(ERR, "Invalid/unsupported asymmetric crypto xform");
572 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
573 __rte_unused uint64_t *opaque,
574 __rte_unused enum qat_device_gen qat_dev_gen)
576 struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
577 struct rte_crypto_asym_op *asym_op = op->asym;
578 struct icp_qat_fw_pke_request *qat_req =
579 (struct icp_qat_fw_pke_request *)out_msg;
580 struct qat_asym_op_cookie *cookie =
581 (struct qat_asym_op_cookie *)op_cookie;
584 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
585 switch (op->sess_type) {
586 case RTE_CRYPTO_OP_WITH_SESSION:
588 "QAT asymmetric crypto PMD does not support session"
591 case RTE_CRYPTO_OP_SESSIONLESS:
592 request_init(qat_req);
593 err = asym_set_input(asym_op, qat_req, cookie,
596 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
601 QAT_DP_LOG(ERR, "Invalid session/xform settings");
602 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
606 qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
607 qat_req->pke_mid.src_data_addr = cookie->input_addr;
608 qat_req->pke_mid.dest_data_addr = cookie->output_addr;
610 HEXDUMP("qat_req:", qat_req, sizeof(struct icp_qat_fw_pke_request));
614 qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
615 HEXDUMP("qat_req:", qat_req, sizeof(struct icp_qat_fw_pke_request));
616 qat_req->output_param_count = 0;
617 qat_req->input_param_count = 0;
618 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
619 cookie->error |= err;
625 qat_asym_collect_response(struct rte_crypto_op *rx_op,
626 struct qat_asym_op_cookie *cookie,
627 struct rte_crypto_asym_xform *xform)
629 struct rte_crypto_asym_op *asym_op = rx_op->asym;
631 switch (xform->xform_type) {
632 case RTE_CRYPTO_ASYM_XFORM_MODEX:
633 return modexp_collect(asym_op, cookie, xform);
634 case RTE_CRYPTO_ASYM_XFORM_MODINV:
635 return modinv_collect(asym_op, cookie, xform);
636 case RTE_CRYPTO_ASYM_XFORM_RSA:
637 return rsa_collect(asym_op, cookie);
639 QAT_LOG(ERR, "Not supported xform type");
640 return RTE_CRYPTO_OP_STATUS_ERROR;
645 qat_asym_process_response(void **op, uint8_t *resp,
646 void *op_cookie, __rte_unused uint64_t *dequeue_err_count)
648 struct icp_qat_fw_pke_resp *resp_msg =
649 (struct icp_qat_fw_pke_resp *)resp;
650 struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
652 struct qat_asym_op_cookie *cookie = op_cookie;
656 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
657 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
658 QAT_DP_LOG(ERR, "Cookie status returned error");
660 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
661 resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
662 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
663 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
664 QAT_DP_LOG(ERR, "Asymmetric response status"
667 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
668 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
669 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
670 QAT_DP_LOG(ERR, "Asymmetric common status"
674 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
675 rx_op->status = qat_asym_collect_response(rx_op,
676 cookie, rx_op->asym->xform);
677 cleanup(cookie, rx_op->asym->xform,
678 cookie->alg_bytesize);
682 HEXDUMP("resp_msg:", resp_msg, sizeof(struct icp_qat_fw_pke_resp));
688 qat_asym_session_configure(struct rte_cryptodev *dev __rte_unused,
689 struct rte_crypto_asym_xform *xform __rte_unused,
690 struct rte_cryptodev_asym_session *sess __rte_unused)
692 QAT_LOG(ERR, "QAT asymmetric PMD currently does not support session");
697 qat_asym_session_get_private_size(struct rte_cryptodev *dev __rte_unused)
699 QAT_LOG(ERR, "QAT asymmetric PMD currently does not support session");
704 qat_asym_session_clear(struct rte_cryptodev *dev __rte_unused,
705 struct rte_cryptodev_asym_session *sess __rte_unused)
707 QAT_LOG(ERR, "QAT asymmetric PMD currently does not support session");
711 qat_asym_crypto_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
714 return qat_enqueue_op_burst(qp, qat_asym_build_request, (void **)ops,
719 qat_asym_crypto_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
722 return qat_dequeue_op_burst(qp, (void **)ops, qat_asym_process_response,
727 qat_asym_init_op_cookie(void *op_cookie)
730 struct qat_asym_op_cookie *cookie = op_cookie;
732 cookie->input_addr = rte_mempool_virt2iova(cookie) +
733 offsetof(struct qat_asym_op_cookie,
736 cookie->output_addr = rte_mempool_virt2iova(cookie) +
737 offsetof(struct qat_asym_op_cookie,
740 for (j = 0; j < 8; j++) {
741 cookie->input_params_ptrs[j] =
742 rte_mempool_virt2iova(cookie) +
743 offsetof(struct qat_asym_op_cookie,
745 cookie->output_params_ptrs[j] =
746 rte_mempool_virt2iova(cookie) +
747 offsetof(struct qat_asym_op_cookie,
753 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
754 struct qat_dev_cmd_param *qat_dev_cmd_param)
756 struct qat_cryptodev_private *internals;
757 struct rte_cryptodev *cryptodev;
758 struct qat_device_info *qat_dev_instance =
759 &qat_pci_devs[qat_pci_dev->qat_dev_id];
760 struct rte_cryptodev_pmd_init_params init_params = {
762 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
763 .private_data_size = sizeof(struct qat_cryptodev_private)
765 struct qat_capabilities_info capa_info;
766 const struct rte_cryptodev_capabilities *capabilities;
767 const struct qat_crypto_gen_dev_ops *gen_dev_ops =
768 &qat_asym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
769 char name[RTE_CRYPTODEV_NAME_MAX_LEN];
770 char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
774 snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
775 qat_pci_dev->name, "asym");
776 QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name);
778 if (gen_dev_ops->cryptodev_ops == NULL) {
779 QAT_LOG(ERR, "Device %s does not support asymmetric crypto",
784 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
785 qat_pci_dev->qat_asym_driver_id =
787 } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
788 if (qat_pci_dev->qat_asym_driver_id !=
789 qat_asym_driver_id) {
791 "Device %s have different driver id than corresponding device in primary process",
797 /* Populate subset device to use in cryptodev device creation */
798 qat_dev_instance->asym_rte_dev.driver = &cryptodev_qat_asym_driver;
799 qat_dev_instance->asym_rte_dev.numa_node =
800 qat_dev_instance->pci_dev->device.numa_node;
801 qat_dev_instance->asym_rte_dev.devargs = NULL;
803 cryptodev = rte_cryptodev_pmd_create(name,
804 &(qat_dev_instance->asym_rte_dev), &init_params);
806 if (cryptodev == NULL)
809 qat_dev_instance->asym_rte_dev.name = cryptodev->data->name;
810 cryptodev->driver_id = qat_asym_driver_id;
811 cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
813 cryptodev->enqueue_burst = qat_asym_crypto_enqueue_op_burst;
814 cryptodev->dequeue_burst = qat_asym_crypto_dequeue_op_burst;
816 cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
818 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
821 snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
822 "QAT_ASYM_CAPA_GEN_%d",
823 qat_pci_dev->qat_dev_gen);
825 internals = cryptodev->data->dev_private;
826 internals->qat_dev = qat_pci_dev;
827 internals->dev_id = cryptodev->data->dev_id;
829 capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
830 capabilities = capa_info.data;
831 capa_size = capa_info.size;
833 internals->capa_mz = rte_memzone_lookup(capa_memz_name);
834 if (internals->capa_mz == NULL) {
835 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
836 capa_size, rte_socket_id(), 0);
837 if (internals->capa_mz == NULL) {
839 "Error allocating memzone for capabilities, "
840 "destroying PMD for %s",
842 rte_cryptodev_pmd_destroy(cryptodev);
843 memset(&qat_dev_instance->asym_rte_dev, 0,
844 sizeof(qat_dev_instance->asym_rte_dev));
849 memcpy(internals->capa_mz->addr, capabilities, capa_size);
850 internals->qat_dev_capabilities = internals->capa_mz->addr;
853 if (qat_dev_cmd_param[i].name == NULL)
855 if (!strcmp(qat_dev_cmd_param[i].name, ASYM_ENQ_THRESHOLD_NAME))
856 internals->min_enq_burst_threshold =
857 qat_dev_cmd_param[i].val;
861 qat_pci_dev->asym_dev = internals;
862 internals->service_type = QAT_SERVICE_ASYMMETRIC;
863 QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
864 cryptodev->data->name, internals->dev_id);
869 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
871 struct rte_cryptodev *cryptodev;
873 if (qat_pci_dev == NULL)
875 if (qat_pci_dev->asym_dev == NULL)
877 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
878 rte_memzone_free(qat_pci_dev->asym_dev->capa_mz);
880 /* free crypto device */
881 cryptodev = rte_cryptodev_pmd_get_dev(
882 qat_pci_dev->asym_dev->dev_id);
883 rte_cryptodev_pmd_destroy(cryptodev);
884 qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
885 qat_pci_dev->asym_dev = NULL;
890 static struct cryptodev_driver qat_crypto_drv;
891 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
892 cryptodev_qat_asym_driver,