1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
8 #include "icp_qat_fw_pke.h"
9 #include "icp_qat_fw.h"
10 #include "qat_pke_functionality_arrays.h"
12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
15 size_t arr_sz, size_t *size, uint32_t *func_id)
19 for (i = 0; i < arr_sz; i++) {
20 if (*size <= arr[i][0]) {
29 static void qat_asym_build_req_tmpl(void *sess_private_data,
30 struct rte_crypto_asym_xform *xform)
33 struct icp_qat_fw_pke_request *qat_req;
34 struct qat_asym_session *session = sess_private_data;
36 qat_req = &session->req_tmpl;
37 memset(qat_req, 0, sizeof(*qat_req));
38 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
40 qat_req->pke_hdr.hdr_flags =
41 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
42 (ICP_QAT_FW_COMN_REQ_FLAG_SET);
43 qat_req->pke_hdr.comn_req_flags = 0;
44 qat_req->pke_hdr.resrvd1 = 0;
45 qat_req->pke_hdr.resrvd2 = 0;
46 qat_req->pke_hdr.kpt_mask = 0;
47 qat_req->pke_hdr.kpt_rn_mask = 0;
48 qat_req->pke_hdr.cd_pars.content_desc_addr = 0;
49 qat_req->pke_hdr.cd_pars.content_desc_resrvd = 0;
52 qat_req->next_req_adr = 0;
54 if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
55 qat_req->output_param_count = 1;
56 qat_req->input_param_count = 3;
57 } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
58 qat_req->output_param_count = 1;
59 qat_req->input_param_count = 2;
63 static size_t max_of(int n, ...)
70 len = va_arg(args, size_t);
72 for (i = 0; i < n - 1; i++) {
73 num = va_arg(args, size_t);
82 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
83 int in_count, int out_count, int in_size, int out_size)
87 for (i = 0; i < in_count; i++)
88 memset(cookie->input_array[i], 0x0, in_size);
89 for (i = 0; i < out_count; i++)
90 memset(cookie->output_array[i], 0x0, out_size);
93 static int qat_asym_check_nonzero(rte_crypto_param n)
96 /* Not a case for any cryptograpic function except for DH
97 * generator which very often can be of one byte length
101 if (n.data[n.length - 1] == 0x0) {
102 for (i = 0; i < n.length - 1; i++)
103 if (n.data[i] != 0x0)
105 if (i == n.length - 1)
106 return QAT_ASYM_ERROR_INVALID_PARAM;
108 } else if (*(uint64_t *)&n.data[
109 n.length - 8] == 0) {
110 /* Very likely it is zeroed modulus */
113 for (i = 0; i < n.length - 8; i++)
114 if (n.data[i] != 0x0)
116 if (i == n.length - 8)
117 return QAT_ASYM_ERROR_INVALID_PARAM;
124 qat_asym_build_request(void *in_op,
127 __rte_unused enum qat_device_gen qat_dev_gen)
129 struct qat_asym_session *ctx;
130 struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
131 struct rte_crypto_asym_op *asym_op = op->asym;
132 struct icp_qat_fw_pke_request *qat_req =
133 (struct icp_qat_fw_pke_request *)out_msg;
134 struct qat_asym_op_cookie *cookie =
135 (struct qat_asym_op_cookie *)op_cookie;
139 size_t alg_size_in_bytes;
142 ctx = (struct qat_asym_session *)get_asym_session_private_data(
143 op->asym->session, cryptodev_qat_asym_driver_id);
144 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
145 qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
147 qat_req->pke_mid.src_data_addr = cookie->input_addr;
148 qat_req->pke_mid.dest_data_addr = cookie->output_addr;
150 if (ctx->alg == QAT_PKE_MODEXP) {
151 err = qat_asym_check_nonzero(ctx->sess_alg_params.mod_exp.n);
153 QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
154 " aborting this operation");
158 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
159 ctx->sess_alg_params.mod_exp.e.length,
160 ctx->sess_alg_params.mod_exp.n.length);
161 alg_size = alg_size_in_bytes << 3;
163 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
164 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
165 &alg_size, &func_id)) {
166 err = QAT_ASYM_ERROR_INVALID_PARAM;
170 alg_size_in_bytes = alg_size >> 3;
171 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
172 asym_op->modex.base.length
173 , asym_op->modex.base.data,
174 asym_op->modex.base.length);
175 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
176 ctx->sess_alg_params.mod_exp.e.length
177 , ctx->sess_alg_params.mod_exp.e.data,
178 ctx->sess_alg_params.mod_exp.e.length);
179 rte_memcpy(cookie->input_array[2] + alg_size_in_bytes -
180 ctx->sess_alg_params.mod_exp.n.length,
181 ctx->sess_alg_params.mod_exp.n.data,
182 ctx->sess_alg_params.mod_exp.n.length);
183 cookie->alg_size = alg_size;
184 qat_req->pke_hdr.cd_pars.func_id = func_id;
185 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
186 QAT_DP_HEXDUMP_LOG(DEBUG, "base",
187 cookie->input_array[0],
189 QAT_DP_HEXDUMP_LOG(DEBUG, "exponent",
190 cookie->input_array[1],
192 QAT_DP_HEXDUMP_LOG(DEBUG, "modulus",
193 cookie->input_array[2],
196 } else if (ctx->alg == QAT_PKE_MODINV) {
197 err = qat_asym_check_nonzero(ctx->sess_alg_params.mod_inv.n);
199 QAT_LOG(ERR, "Empty modulus in modular multiplicative"
200 " inverse, aborting this operation");
204 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
205 ctx->sess_alg_params.mod_inv.n.length);
206 alg_size = alg_size_in_bytes << 3;
208 if (ctx->sess_alg_params.mod_inv.n.data[
209 ctx->sess_alg_params.mod_inv.n.length - 1] & 0x01) {
210 if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
211 sizeof(MOD_INV_IDS_ODD)/
212 sizeof(*MOD_INV_IDS_ODD),
213 &alg_size, &func_id)) {
214 err = QAT_ASYM_ERROR_INVALID_PARAM;
218 if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
219 sizeof(MOD_INV_IDS_EVEN)/
220 sizeof(*MOD_INV_IDS_EVEN),
221 &alg_size, &func_id)) {
222 err = QAT_ASYM_ERROR_INVALID_PARAM;
227 alg_size_in_bytes = alg_size >> 3;
228 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
229 asym_op->modinv.base.length
230 , asym_op->modinv.base.data,
231 asym_op->modinv.base.length);
232 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
233 ctx->sess_alg_params.mod_inv.n.length
234 , ctx->sess_alg_params.mod_inv.n.data,
235 ctx->sess_alg_params.mod_inv.n.length);
236 cookie->alg_size = alg_size;
237 qat_req->pke_hdr.cd_pars.func_id = func_id;
238 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
239 QAT_DP_HEXDUMP_LOG(DEBUG, "base",
240 cookie->input_array[0],
242 QAT_DP_HEXDUMP_LOG(DEBUG, "modulus",
243 cookie->input_array[1],
248 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
249 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
250 sizeof(struct icp_qat_fw_pke_request));
254 qat_req->output_param_count = 0;
255 qat_req->input_param_count = 0;
256 qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
257 cookie->error |= err;
263 qat_asym_process_response(void **op, uint8_t *resp,
266 struct qat_asym_session *ctx;
267 struct icp_qat_fw_pke_resp *resp_msg =
268 (struct icp_qat_fw_pke_resp *)resp;
269 struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
271 struct rte_crypto_asym_op *asym_op = rx_op->asym;
272 struct qat_asym_op_cookie *cookie = op_cookie;
273 size_t alg_size, alg_size_in_bytes;
275 ctx = (struct qat_asym_session *)get_asym_session_private_data(
276 rx_op->asym->session, cryptodev_qat_asym_driver_id);
279 rx_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
283 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
284 QAT_DP_LOG(ERR, "Cookie status returned error");
286 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
287 resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
288 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
289 QAT_DP_LOG(ERR, "Asymmetric response status returned error");
291 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
292 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
293 QAT_DP_LOG(ERR, "Asymmetric common status returned error");
297 if (ctx->alg == QAT_PKE_MODEXP) {
298 alg_size = cookie->alg_size;
299 alg_size_in_bytes = alg_size >> 3;
300 uint8_t *modexp_result = asym_op->modex.result.data;
302 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
303 rte_memcpy(modexp_result +
304 (asym_op->modex.result.length -
305 ctx->sess_alg_params.mod_exp.n.length),
306 cookie->output_array[0] + alg_size_in_bytes
307 - ctx->sess_alg_params.mod_exp.n.length,
308 ctx->sess_alg_params.mod_exp.n.length
310 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
311 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
312 QAT_DP_HEXDUMP_LOG(DEBUG, "modexp_result",
313 cookie->output_array[0],
317 qat_clear_arrays(cookie, 3, 1, alg_size_in_bytes,
319 } else if (ctx->alg == QAT_PKE_MODINV) {
320 alg_size = cookie->alg_size;
321 alg_size_in_bytes = alg_size >> 3;
322 uint8_t *modinv_result = asym_op->modinv.result.data;
324 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
325 rte_memcpy(modinv_result + (asym_op->modinv.result.length
326 - ctx->sess_alg_params.mod_inv.n.length),
327 cookie->output_array[0] + alg_size_in_bytes
328 - ctx->sess_alg_params.mod_inv.n.length,
329 ctx->sess_alg_params.mod_inv.n.length);
330 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
331 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
332 QAT_DP_HEXDUMP_LOG(DEBUG, "modinv_result",
333 cookie->output_array[0],
337 qat_clear_arrays(cookie, 2, 1, alg_size_in_bytes,
341 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
342 QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
343 sizeof(struct icp_qat_fw_pke_resp));
348 qat_asym_session_configure(struct rte_cryptodev *dev,
349 struct rte_crypto_asym_xform *xform,
350 struct rte_cryptodev_asym_session *sess,
351 struct rte_mempool *mempool)
354 void *sess_private_data;
355 struct qat_asym_session *session;
357 if (rte_mempool_get(mempool, &sess_private_data)) {
359 "Couldn't get object from session mempool");
363 session = sess_private_data;
364 if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
365 session->sess_alg_params.mod_exp.e = xform->modex.exponent;
366 session->sess_alg_params.mod_exp.n = xform->modex.modulus;
367 session->alg = QAT_PKE_MODEXP;
369 if (xform->modex.exponent.length == 0 ||
370 xform->modex.modulus.length == 0) {
371 QAT_LOG(ERR, "Invalid mod exp input parameter");
375 } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
376 session->sess_alg_params.mod_inv.n = xform->modinv.modulus;
377 session->alg = QAT_PKE_MODINV;
379 if (xform->modinv.modulus.length == 0) {
380 QAT_LOG(ERR, "Invalid mod inv input parameter");
385 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
389 qat_asym_build_req_tmpl(sess_private_data, xform);
390 set_asym_session_private_data(sess, dev->driver_id,
395 rte_mempool_put(mempool, sess_private_data);
399 unsigned int qat_asym_session_get_private_size(
400 struct rte_cryptodev *dev __rte_unused)
402 return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
406 qat_asym_session_clear(struct rte_cryptodev *dev,
407 struct rte_cryptodev_asym_session *sess)
409 uint8_t index = dev->driver_id;
410 void *sess_priv = get_asym_session_private_data(sess, index);
411 struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
414 memset(s, 0, qat_asym_session_get_private_size(dev));
415 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
417 set_asym_session_private_data(sess, index, NULL);
418 rte_mempool_put(sess_mp, sess_priv);