1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
8 #include "icp_qat_fw_pke.h"
9 #include "icp_qat_fw.h"
10 #include "qat_pke_functionality_arrays.h"
12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
15 size_t arr_sz, size_t *size, uint32_t *func_id)
19 for (i = 0; i < arr_sz; i++) {
20 if (*size <= arr[i][0]) {
29 static inline void qat_fill_req_tmpl(struct icp_qat_fw_pke_request *qat_req)
31 memset(qat_req, 0, sizeof(*qat_req));
32 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
34 qat_req->pke_hdr.hdr_flags =
35 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
36 (ICP_QAT_FW_COMN_REQ_FLAG_SET);
39 static inline void qat_asym_build_req_tmpl(void *sess_private_data)
41 struct icp_qat_fw_pke_request *qat_req;
42 struct qat_asym_session *session = sess_private_data;
44 qat_req = &session->req_tmpl;
45 qat_fill_req_tmpl(qat_req);
48 static size_t max_of(int n, ...)
55 len = va_arg(args, size_t);
57 for (i = 0; i < n - 1; i++) {
58 num = va_arg(args, size_t);
67 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
68 int in_count, int out_count, int in_size, int out_size)
72 for (i = 0; i < in_count; i++)
73 memset(cookie->input_array[i], 0x0, in_size);
74 for (i = 0; i < out_count; i++)
75 memset(cookie->output_array[i], 0x0, out_size);
78 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
79 enum rte_crypto_asym_xform_type alg, int in_size, int out_size)
81 if (alg == RTE_CRYPTO_ASYM_XFORM_MODEX)
82 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
83 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, in_size,
85 else if (alg == RTE_CRYPTO_ASYM_XFORM_MODINV)
86 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
87 QAT_ASYM_MODINV_NUM_OUT_PARAMS, in_size,
91 static int qat_asym_check_nonzero(rte_crypto_param n)
94 /* Not a case for any cryptograpic function except for DH
95 * generator which very often can be of one byte length
99 if (n.data[n.length - 1] == 0x0) {
100 for (i = 0; i < n.length - 1; i++)
101 if (n.data[i] != 0x0)
103 if (i == n.length - 1)
106 } else if (*(uint64_t *)&n.data[
107 n.length - 8] == 0) {
108 /* Very likely it is zeroed modulus */
111 for (i = 0; i < n.length - 8; i++)
112 if (n.data[i] != 0x0)
114 if (i == n.length - 8)
122 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
123 struct icp_qat_fw_pke_request *qat_req,
124 struct qat_asym_op_cookie *cookie,
125 struct rte_crypto_asym_xform *xform)
129 size_t alg_size_in_bytes;
130 uint32_t func_id = 0;
132 if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
133 err = qat_asym_check_nonzero(xform->modex.modulus);
135 QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
136 " aborting this operation");
140 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
141 xform->modex.exponent.length,
142 xform->modex.modulus.length);
143 alg_size = alg_size_in_bytes << 3;
145 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
146 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
147 &alg_size, &func_id)) {
151 alg_size_in_bytes = alg_size >> 3;
152 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
153 asym_op->modex.base.length
154 , asym_op->modex.base.data,
155 asym_op->modex.base.length);
156 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
157 xform->modex.exponent.length
158 , xform->modex.exponent.data,
159 xform->modex.exponent.length);
160 rte_memcpy(cookie->input_array[2] + alg_size_in_bytes -
161 xform->modex.modulus.length,
162 xform->modex.modulus.data,
163 xform->modex.modulus.length);
164 cookie->alg_size = alg_size;
165 qat_req->pke_hdr.cd_pars.func_id = func_id;
166 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
167 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
168 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
169 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
170 cookie->input_array[0],
172 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
173 cookie->input_array[1],
175 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
176 cookie->input_array[2],
179 } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
180 err = qat_asym_check_nonzero(xform->modinv.modulus);
182 QAT_LOG(ERR, "Empty modulus in modular multiplicative"
183 " inverse, aborting this operation");
187 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
188 xform->modinv.modulus.length);
189 alg_size = alg_size_in_bytes << 3;
191 if (xform->modinv.modulus.data[
192 xform->modinv.modulus.length - 1] & 0x01) {
193 if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
194 sizeof(MOD_INV_IDS_ODD)/
195 sizeof(*MOD_INV_IDS_ODD),
196 &alg_size, &func_id)) {
200 if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
201 sizeof(MOD_INV_IDS_EVEN)/
202 sizeof(*MOD_INV_IDS_EVEN),
203 &alg_size, &func_id)) {
208 alg_size_in_bytes = alg_size >> 3;
209 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
210 asym_op->modinv.base.length
211 , asym_op->modinv.base.data,
212 asym_op->modinv.base.length);
213 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
214 xform->modinv.modulus.length
215 , xform->modinv.modulus.data,
216 xform->modinv.modulus.length);
217 cookie->alg_size = alg_size;
218 qat_req->pke_hdr.cd_pars.func_id = func_id;
219 qat_req->input_param_count =
220 QAT_ASYM_MODINV_NUM_IN_PARAMS;
221 qat_req->output_param_count =
222 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
223 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
224 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
225 cookie->input_array[0],
227 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
228 cookie->input_array[1],
232 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
239 qat_asym_build_request(void *in_op,
242 __rte_unused enum qat_device_gen qat_dev_gen)
244 struct qat_asym_session *ctx;
245 struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
246 struct rte_crypto_asym_op *asym_op = op->asym;
247 struct icp_qat_fw_pke_request *qat_req =
248 (struct icp_qat_fw_pke_request *)out_msg;
249 struct qat_asym_op_cookie *cookie =
250 (struct qat_asym_op_cookie *)op_cookie;
253 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
254 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
255 ctx = (struct qat_asym_session *)
256 get_asym_session_private_data(
257 op->asym->session, cryptodev_qat_asym_driver_id);
258 if (unlikely(ctx == NULL)) {
259 QAT_LOG(ERR, "Session has not been created for this device");
262 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
263 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
265 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
268 } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
269 qat_fill_req_tmpl(qat_req);
270 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
273 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
277 QAT_DP_LOG(ERR, "Invalid session/xform settings");
278 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
282 qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
283 qat_req->pke_mid.src_data_addr = cookie->input_addr;
284 qat_req->pke_mid.dest_data_addr = cookie->output_addr;
286 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
287 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
288 sizeof(struct icp_qat_fw_pke_request));
294 qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
296 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
297 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
298 sizeof(struct icp_qat_fw_pke_request));
301 qat_req->output_param_count = 0;
302 qat_req->input_param_count = 0;
303 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
304 cookie->error |= err;
309 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
310 struct qat_asym_op_cookie *cookie,
311 struct rte_crypto_asym_xform *xform)
313 size_t alg_size, alg_size_in_bytes = 0;
314 struct rte_crypto_asym_op *asym_op = rx_op->asym;
316 if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
317 rte_crypto_param n = xform->modex.modulus;
319 alg_size = cookie->alg_size;
320 alg_size_in_bytes = alg_size >> 3;
321 uint8_t *modexp_result = asym_op->modex.result.data;
323 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
324 rte_memcpy(modexp_result +
325 (asym_op->modex.result.length -
327 cookie->output_array[0] + alg_size_in_bytes
330 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
331 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
332 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
333 cookie->output_array[0],
338 } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
339 rte_crypto_param n = xform->modinv.modulus;
341 alg_size = cookie->alg_size;
342 alg_size_in_bytes = alg_size >> 3;
343 uint8_t *modinv_result = asym_op->modinv.result.data;
345 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
346 rte_memcpy(modinv_result + (asym_op->modinv.result.length
348 cookie->output_array[0] + alg_size_in_bytes
349 - n.length, n.length);
350 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
351 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
352 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
353 cookie->output_array[0],
358 qat_clear_arrays_by_alg(cookie, xform->xform_type, alg_size_in_bytes,
363 qat_asym_process_response(void **op, uint8_t *resp,
366 struct qat_asym_session *ctx;
367 struct icp_qat_fw_pke_resp *resp_msg =
368 (struct icp_qat_fw_pke_resp *)resp;
369 struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
371 struct qat_asym_op_cookie *cookie = op_cookie;
375 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
376 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
377 QAT_DP_LOG(ERR, "Cookie status returned error");
379 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
380 resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
381 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
382 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
383 QAT_DP_LOG(ERR, "Asymmetric response status"
386 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
387 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
388 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
389 QAT_DP_LOG(ERR, "Asymmetric common status"
394 if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
395 ctx = (struct qat_asym_session *)get_asym_session_private_data(
396 rx_op->asym->session, cryptodev_qat_asym_driver_id);
397 qat_asym_collect_response(rx_op, cookie, ctx->xform);
398 } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
399 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
403 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
404 QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
405 sizeof(struct icp_qat_fw_pke_resp));
410 qat_asym_session_configure(struct rte_cryptodev *dev,
411 struct rte_crypto_asym_xform *xform,
412 struct rte_cryptodev_asym_session *sess,
413 struct rte_mempool *mempool)
416 void *sess_private_data;
417 struct qat_asym_session *session;
419 if (rte_mempool_get(mempool, &sess_private_data)) {
421 "Couldn't get object from session mempool");
425 session = sess_private_data;
426 if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
427 if (xform->modex.exponent.length == 0 ||
428 xform->modex.modulus.length == 0) {
429 QAT_LOG(ERR, "Invalid mod exp input parameter");
433 } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
434 if (xform->modinv.modulus.length == 0) {
435 QAT_LOG(ERR, "Invalid mod inv input parameter");
439 } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
440 || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
441 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
445 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
450 session->xform = xform;
451 qat_asym_build_req_tmpl(sess_private_data);
452 set_asym_session_private_data(sess, dev->driver_id,
457 rte_mempool_put(mempool, sess_private_data);
461 unsigned int qat_asym_session_get_private_size(
462 struct rte_cryptodev *dev __rte_unused)
464 return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
468 qat_asym_session_clear(struct rte_cryptodev *dev,
469 struct rte_cryptodev_asym_session *sess)
471 uint8_t index = dev->driver_id;
472 void *sess_priv = get_asym_session_private_data(sess, index);
473 struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
476 memset(s, 0, qat_asym_session_get_private_size(dev));
477 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
479 set_asym_session_private_data(sess, index, NULL);
480 rte_mempool_put(sess_mp, sess_priv);