crypto/octeontx2: move capabilities init into probe
[dpdk.git] / drivers / crypto / qat / qat_asym.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2019 Intel Corporation
3  */
4
5 #include <stdarg.h>
6
7 #include "qat_asym.h"
8 #include "icp_qat_fw_pke.h"
9 #include "icp_qat_fw.h"
10 #include "qat_pke_functionality_arrays.h"
11
12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
13
14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
15                 size_t arr_sz, size_t *size, uint32_t *func_id)
16 {
17         size_t i;
18
19         for (i = 0; i < arr_sz; i++) {
20                 if (*size <= arr[i][0]) {
21                         *size = arr[i][0];
22                         *func_id = arr[i][1];
23                         return 0;
24                 }
25         }
26         return -1;
27 }
28
29 static inline void qat_fill_req_tmpl(struct icp_qat_fw_pke_request *qat_req)
30 {
31         memset(qat_req, 0, sizeof(*qat_req));
32         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
33
34         qat_req->pke_hdr.hdr_flags =
35                         ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
36                         (ICP_QAT_FW_COMN_REQ_FLAG_SET);
37 }
38
39 static inline void qat_asym_build_req_tmpl(void *sess_private_data)
40 {
41         struct icp_qat_fw_pke_request *qat_req;
42         struct qat_asym_session *session = sess_private_data;
43
44         qat_req = &session->req_tmpl;
45         qat_fill_req_tmpl(qat_req);
46 }
47
48 static size_t max_of(int n, ...)
49 {
50         va_list args;
51         size_t len = 0, num;
52         int i;
53
54         va_start(args, n);
55         len = va_arg(args, size_t);
56
57         for (i = 0; i < n - 1; i++) {
58                 num = va_arg(args, size_t);
59                 if (num > len)
60                         len = num;
61         }
62         va_end(args);
63
64         return len;
65 }
66
67 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
68                 int in_count, int out_count, int in_size, int out_size)
69 {
70         int i;
71
72         for (i = 0; i < in_count; i++)
73                 memset(cookie->input_array[i], 0x0, in_size);
74         for (i = 0; i < out_count; i++)
75                 memset(cookie->output_array[i], 0x0, out_size);
76 }
77
78 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
79                 enum rte_crypto_asym_xform_type alg, int in_size, int out_size)
80 {
81         if (alg == RTE_CRYPTO_ASYM_XFORM_MODEX)
82                 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
83                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, in_size,
84                                 out_size);
85         else if (alg == RTE_CRYPTO_ASYM_XFORM_MODINV)
86                 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
87                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, in_size,
88                                 out_size);
89 }
90
91 static int qat_asym_check_nonzero(rte_crypto_param n)
92 {
93         if (n.length < 8) {
94                 /* Not a case for any cryptograpic function except for DH
95                  * generator which very often can be of one byte length
96                  */
97                 size_t i;
98
99                 if (n.data[n.length - 1] == 0x0) {
100                         for (i = 0; i < n.length - 1; i++)
101                                 if (n.data[i] != 0x0)
102                                         break;
103                         if (i == n.length - 1)
104                                 return -(EINVAL);
105                 }
106         } else if (*(uint64_t *)&n.data[
107                                 n.length - 8] == 0) {
108                 /* Very likely it is zeroed modulus */
109                 size_t i;
110
111                 for (i = 0; i < n.length - 8; i++)
112                         if (n.data[i] != 0x0)
113                                 break;
114                 if (i == n.length - 8)
115                         return -(EINVAL);
116         }
117
118         return 0;
119 }
120
121 static int
122 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
123                 struct icp_qat_fw_pke_request *qat_req,
124                 struct qat_asym_op_cookie *cookie,
125                 struct rte_crypto_asym_xform *xform)
126 {
127         int err = 0;
128         size_t alg_size;
129         size_t alg_size_in_bytes;
130         uint32_t func_id = 0;
131
132         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
133                 err = qat_asym_check_nonzero(xform->modex.modulus);
134                 if (err) {
135                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
136                                         " aborting this operation");
137                         return err;
138                 }
139
140                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
141                                xform->modex.exponent.length,
142                                xform->modex.modulus.length);
143                 alg_size = alg_size_in_bytes << 3;
144
145                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
146                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
147                                 &alg_size, &func_id)) {
148                         return -(EINVAL);
149                 }
150
151                 alg_size_in_bytes = alg_size >> 3;
152                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
153                         asym_op->modex.base.length
154                         , asym_op->modex.base.data,
155                         asym_op->modex.base.length);
156                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
157                         xform->modex.exponent.length
158                         , xform->modex.exponent.data,
159                         xform->modex.exponent.length);
160                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
161                         xform->modex.modulus.length,
162                         xform->modex.modulus.data,
163                         xform->modex.modulus.length);
164                 cookie->alg_size = alg_size;
165                 qat_req->pke_hdr.cd_pars.func_id = func_id;
166                 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
167                 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
168 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
169                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
170                                 cookie->input_array[0],
171                                 alg_size_in_bytes);
172                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
173                                 cookie->input_array[1],
174                                 alg_size_in_bytes);
175                 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
176                                 cookie->input_array[2],
177                                 alg_size_in_bytes);
178 #endif
179         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
180                 err = qat_asym_check_nonzero(xform->modinv.modulus);
181                 if (err) {
182                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
183                                         " inverse, aborting this operation");
184                         return err;
185                 }
186
187                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
188                                 xform->modinv.modulus.length);
189                 alg_size = alg_size_in_bytes << 3;
190
191                 if (xform->modinv.modulus.data[
192                                 xform->modinv.modulus.length - 1] & 0x01) {
193                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
194                                         sizeof(MOD_INV_IDS_ODD)/
195                                         sizeof(*MOD_INV_IDS_ODD),
196                                         &alg_size, &func_id)) {
197                                 return -(EINVAL);
198                         }
199                 } else {
200                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
201                                         sizeof(MOD_INV_IDS_EVEN)/
202                                         sizeof(*MOD_INV_IDS_EVEN),
203                                         &alg_size, &func_id)) {
204                                 return -(EINVAL);
205                         }
206                 }
207
208                 alg_size_in_bytes = alg_size >> 3;
209                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
210                         asym_op->modinv.base.length
211                                 , asym_op->modinv.base.data,
212                                 asym_op->modinv.base.length);
213                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
214                                 xform->modinv.modulus.length
215                                 , xform->modinv.modulus.data,
216                                 xform->modinv.modulus.length);
217                 cookie->alg_size = alg_size;
218                 qat_req->pke_hdr.cd_pars.func_id = func_id;
219                 qat_req->input_param_count =
220                                 QAT_ASYM_MODINV_NUM_IN_PARAMS;
221                 qat_req->output_param_count =
222                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
223 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
224                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
225                                 cookie->input_array[0],
226                                 alg_size_in_bytes);
227                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
228                                 cookie->input_array[1],
229                                 alg_size_in_bytes);
230 #endif
231         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
232                 err = qat_asym_check_nonzero(xform->rsa.n);
233                 if (err) {
234                         QAT_LOG(ERR, "Empty modulus in RSA"
235                                         " inverse, aborting this operation");
236                         return err;
237                 }
238
239                 alg_size_in_bytes = xform->rsa.n.length;
240                 alg_size = alg_size_in_bytes << 3;
241
242                 qat_req->input_param_count =
243                                 QAT_ASYM_RSA_NUM_IN_PARAMS;
244                 qat_req->output_param_count =
245                                 QAT_ASYM_RSA_NUM_OUT_PARAMS;
246
247                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
248                                 asym_op->rsa.op_type ==
249                                                 RTE_CRYPTO_ASYM_OP_VERIFY) {
250
251                         if (qat_asym_get_sz_and_func_id(RSA_ENC_IDS,
252                                         sizeof(RSA_ENC_IDS)/
253                                         sizeof(*RSA_ENC_IDS),
254                                         &alg_size, &func_id)) {
255                                 err = -(EINVAL);
256                                 QAT_LOG(ERR,
257                                         "Not supported RSA parameter size (key)");
258                                 return err;
259                         }
260                         alg_size_in_bytes = alg_size >> 3;
261                         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
262                                 switch (asym_op->rsa.pad) {
263                                 case RTE_CRYPTO_RSA_PADDING_NONE:
264                                         rte_memcpy(cookie->input_array[0] +
265                                                 alg_size_in_bytes -
266                                                 asym_op->rsa.message.length
267                                                 , asym_op->rsa.message.data,
268                                                 asym_op->rsa.message.length);
269                                         break;
270                                 default:
271                                         err = -(EINVAL);
272                                         QAT_LOG(ERR,
273                                                 "Invalid RSA padding (Encryption)");
274                                         return err;
275                                 }
276 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
277                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Message",
278                                                 cookie->input_array[0],
279                                                 alg_size_in_bytes);
280 #endif
281                         } else {
282                                 switch (asym_op->rsa.pad) {
283                                 case RTE_CRYPTO_RSA_PADDING_NONE:
284                                         rte_memcpy(cookie->input_array[0],
285                                                 asym_op->rsa.sign.data,
286                                                 alg_size_in_bytes);
287                                         break;
288                                 default:
289                                         err = -(EINVAL);
290                                         QAT_LOG(ERR,
291                                                 "Invalid RSA padding (Verify)");
292                                         return err;
293                                 }
294
295 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
296                                 QAT_DP_HEXDUMP_LOG(DEBUG, " RSA Signature",
297                                                 cookie->input_array[0],
298                                                 alg_size_in_bytes);
299 #endif
300
301                         }
302                         rte_memcpy(cookie->input_array[1] +
303                                         alg_size_in_bytes -
304                                         xform->rsa.e.length
305                                         , xform->rsa.e.data,
306                                         xform->rsa.e.length);
307                         rte_memcpy(cookie->input_array[2] +
308                                         alg_size_in_bytes -
309                                         xform->rsa.n.length,
310                                         xform->rsa.n.data,
311                                         xform->rsa.n.length);
312
313                         cookie->alg_size = alg_size;
314                         qat_req->pke_hdr.cd_pars.func_id = func_id;
315
316 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
317                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Public Key",
318                                         cookie->input_array[1], alg_size_in_bytes);
319                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Modulus",
320                                         cookie->input_array[2], alg_size_in_bytes);
321 #endif
322                 } else {
323                         if (asym_op->rsa.op_type ==
324                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
325                                 switch (asym_op->rsa.pad) {
326                                 case RTE_CRYPTO_RSA_PADDING_NONE:
327                                         rte_memcpy(cookie->input_array[0]
328                                                 + alg_size_in_bytes -
329                                                 asym_op->rsa.cipher.length,
330                                                 asym_op->rsa.cipher.data,
331                                                 asym_op->rsa.cipher.length);
332                                         break;
333                                 default:
334                                         QAT_LOG(ERR,
335                                                 "Invalid padding of RSA (Decrypt)");
336                                         return -(EINVAL);
337                                 }
338
339                         } else if (asym_op->rsa.op_type ==
340                                         RTE_CRYPTO_ASYM_OP_SIGN) {
341                                 switch (asym_op->rsa.pad) {
342                                 case RTE_CRYPTO_RSA_PADDING_NONE:
343                                         rte_memcpy(cookie->input_array[0]
344                                                 + alg_size_in_bytes -
345                                                 asym_op->rsa.message.length,
346                                                 asym_op->rsa.message.data,
347                                                 asym_op->rsa.message.length);
348                                         break;
349                                 default:
350                                         QAT_LOG(ERR,
351                                                 "Invalid padding of RSA (Signature)");
352                                         return -(EINVAL);
353                                 }
354                         }
355                         if (xform->rsa.key_type == RTE_RSA_KET_TYPE_QT) {
356
357                                 qat_req->input_param_count =
358                                                 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
359                                 if (qat_asym_get_sz_and_func_id(RSA_DEC_CRT_IDS,
360                                                 sizeof(RSA_DEC_CRT_IDS)/
361                                                 sizeof(*RSA_DEC_CRT_IDS),
362                                                 &alg_size, &func_id)) {
363                                         return -(EINVAL);
364                                 }
365                                 alg_size_in_bytes = alg_size >> 3;
366
367                                 rte_memcpy(cookie->input_array[1] +
368                                                 (alg_size_in_bytes >> 1) -
369                                                 xform->rsa.qt.p.length
370                                                 , xform->rsa.qt.p.data,
371                                                 xform->rsa.qt.p.length);
372                                 rte_memcpy(cookie->input_array[2] +
373                                                 (alg_size_in_bytes >> 1) -
374                                                 xform->rsa.qt.q.length
375                                                 , xform->rsa.qt.q.data,
376                                                 xform->rsa.qt.q.length);
377                                 rte_memcpy(cookie->input_array[3] +
378                                                 (alg_size_in_bytes >> 1) -
379                                                 xform->rsa.qt.dP.length
380                                                 , xform->rsa.qt.dP.data,
381                                                 xform->rsa.qt.dP.length);
382                                 rte_memcpy(cookie->input_array[4] +
383                                                 (alg_size_in_bytes >> 1) -
384                                                 xform->rsa.qt.dQ.length
385                                                 , xform->rsa.qt.dQ.data,
386                                                 xform->rsa.qt.dQ.length);
387                                 rte_memcpy(cookie->input_array[5] +
388                                                 (alg_size_in_bytes >> 1) -
389                                                 xform->rsa.qt.qInv.length
390                                                 , xform->rsa.qt.qInv.data,
391                                                 xform->rsa.qt.qInv.length);
392                                 cookie->alg_size = alg_size;
393                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
394
395 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
396                                 QAT_DP_HEXDUMP_LOG(DEBUG, "C",
397                                                 cookie->input_array[0],
398                                                 alg_size_in_bytes);
399                                 QAT_DP_HEXDUMP_LOG(DEBUG, "p",
400                                                 cookie->input_array[1],
401                                                 alg_size_in_bytes);
402                                 QAT_DP_HEXDUMP_LOG(DEBUG, "q",
403                                                 cookie->input_array[2],
404                                                 alg_size_in_bytes);
405                                 QAT_DP_HEXDUMP_LOG(DEBUG,
406                                                 "dP", cookie->input_array[3],
407                                                 alg_size_in_bytes);
408                                 QAT_DP_HEXDUMP_LOG(DEBUG,
409                                                 "dQ", cookie->input_array[4],
410                                                 alg_size_in_bytes);
411                                 QAT_DP_HEXDUMP_LOG(DEBUG,
412                                                 "qInv", cookie->input_array[5],
413                                                 alg_size_in_bytes);
414 #endif
415                         } else if (xform->rsa.key_type ==
416                                         RTE_RSA_KEY_TYPE_EXP) {
417                                 if (qat_asym_get_sz_and_func_id(
418                                                 RSA_DEC_IDS,
419                                                 sizeof(RSA_DEC_IDS)/
420                                                 sizeof(*RSA_DEC_IDS),
421                                                 &alg_size, &func_id)) {
422                                         return -(EINVAL);
423                                 }
424                                 alg_size_in_bytes = alg_size >> 3;
425                                 rte_memcpy(cookie->input_array[1] +
426                                                 alg_size_in_bytes -
427                                                 xform->rsa.d.length,
428                                                 xform->rsa.d.data,
429                                                 xform->rsa.d.length);
430                                 rte_memcpy(cookie->input_array[2] +
431                                                 alg_size_in_bytes -
432                                                 xform->rsa.n.length,
433                                                 xform->rsa.n.data,
434                                                 xform->rsa.n.length);
435 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
436                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA ciphertext",
437                                         cookie->input_array[0],
438                                         alg_size_in_bytes);
439                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA d", cookie->input_array[1],
440                                         alg_size_in_bytes);
441                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA n", cookie->input_array[2],
442                                         alg_size_in_bytes);
443 #endif
444
445                                 cookie->alg_size = alg_size;
446                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
447                         } else {
448                                 QAT_LOG(ERR, "Invalid RSA key type");
449                                 return -(EINVAL);
450                         }
451                 }
452         } else {
453                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
454                 return -(EINVAL);
455         }
456         return 0;
457 }
458
459 int
460 qat_asym_build_request(void *in_op,
461                         uint8_t *out_msg,
462                         void *op_cookie,
463                         __rte_unused enum qat_device_gen qat_dev_gen)
464 {
465         struct qat_asym_session *ctx;
466         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
467         struct rte_crypto_asym_op *asym_op = op->asym;
468         struct icp_qat_fw_pke_request *qat_req =
469                         (struct icp_qat_fw_pke_request *)out_msg;
470         struct qat_asym_op_cookie *cookie =
471                                 (struct qat_asym_op_cookie *)op_cookie;
472         int err = 0;
473
474         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
475         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
476                 ctx = (struct qat_asym_session *)
477                         get_asym_session_private_data(
478                         op->asym->session, qat_asym_driver_id);
479                 if (unlikely(ctx == NULL)) {
480                         QAT_LOG(ERR, "Session has not been created for this device");
481                         goto error;
482                 }
483                 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
484                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
485                 if (err) {
486                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
487                         goto error;
488                 }
489         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
490                 qat_fill_req_tmpl(qat_req);
491                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
492                                 op->asym->xform);
493                 if (err) {
494                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
495                         goto error;
496                 }
497         } else {
498                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
499                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
500                 goto error;
501         }
502
503         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
504         qat_req->pke_mid.src_data_addr = cookie->input_addr;
505         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
506
507 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
508         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
509                         sizeof(struct icp_qat_fw_pke_request));
510 #endif
511
512         return 0;
513 error:
514
515         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
516
517 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
518         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
519                 sizeof(struct icp_qat_fw_pke_request));
520 #endif
521
522         qat_req->output_param_count = 0;
523         qat_req->input_param_count = 0;
524         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
525         cookie->error |= err;
526
527         return 0;
528 }
529
530 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
531                 struct qat_asym_op_cookie *cookie,
532                 struct rte_crypto_asym_xform *xform)
533 {
534         size_t alg_size, alg_size_in_bytes = 0;
535         struct rte_crypto_asym_op *asym_op = rx_op->asym;
536
537         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
538                 rte_crypto_param n = xform->modex.modulus;
539
540                 alg_size = cookie->alg_size;
541                 alg_size_in_bytes = alg_size >> 3;
542                 uint8_t *modexp_result = asym_op->modex.result.data;
543
544                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
545                         rte_memcpy(modexp_result +
546                                 (asym_op->modex.result.length -
547                                         n.length),
548                                 cookie->output_array[0] + alg_size_in_bytes
549                                 - n.length, n.length
550                                 );
551                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
552 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
553                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
554                                         cookie->output_array[0],
555                                         alg_size_in_bytes);
556
557 #endif
558                 }
559         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
560                 rte_crypto_param n = xform->modinv.modulus;
561
562                 alg_size = cookie->alg_size;
563                 alg_size_in_bytes = alg_size >> 3;
564                 uint8_t *modinv_result = asym_op->modinv.result.data;
565
566                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
567                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
568                                 - n.length),
569                                 cookie->output_array[0] + alg_size_in_bytes
570                                 - n.length, n.length);
571                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
572 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
573                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
574                                         cookie->output_array[0],
575                                         alg_size_in_bytes);
576 #endif
577                 }
578         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
579
580                 alg_size = cookie->alg_size;
581                 alg_size_in_bytes = alg_size >> 3;
582                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
583                                 asym_op->rsa.op_type ==
584                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
585                         if (asym_op->rsa.op_type ==
586                                         RTE_CRYPTO_ASYM_OP_ENCRYPT) {
587                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
588
589                                 rte_memcpy(rsa_result,
590                                                 cookie->output_array[0],
591                                                 alg_size_in_bytes);
592                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
593 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
594                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Encrypted data",
595                                                 cookie->output_array[0],
596                                                 alg_size_in_bytes);
597 #endif
598                         } else if (asym_op->rsa.op_type ==
599                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
600                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
601
602                                 switch (asym_op->rsa.pad) {
603                                 case RTE_CRYPTO_RSA_PADDING_NONE:
604                                         rte_memcpy(rsa_result,
605                                                         cookie->output_array[0],
606                                                         alg_size_in_bytes);
607                                         rx_op->status =
608                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
609 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
610                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
611                                                 cookie->output_array[0],
612                                                 alg_size_in_bytes);
613 #endif
614                                         break;
615                                 default:
616                                         QAT_LOG(ERR, "Padding not supported");
617                                         rx_op->status =
618                                                 RTE_CRYPTO_OP_STATUS_ERROR;
619                                         break;
620                                 }
621                         }
622                 } else {
623                         if (asym_op->rsa.op_type ==
624                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
625                                 uint8_t *rsa_result = asym_op->rsa.message.data;
626
627                                 switch (asym_op->rsa.pad) {
628                                 case RTE_CRYPTO_RSA_PADDING_NONE:
629                                         rte_memcpy(rsa_result,
630                                                 cookie->output_array[0],
631                                                 alg_size_in_bytes);
632                                         break;
633                                 default:
634                                         QAT_LOG(ERR, "Padding not supported");
635                                         rx_op->status =
636                                                 RTE_CRYPTO_OP_STATUS_ERROR;
637                                         break;
638                                 }
639 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
640                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Decrypted Message",
641                                                 rsa_result, alg_size_in_bytes);
642 #endif
643                         } else if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
644                                 uint8_t *rsa_result = asym_op->rsa.sign.data;
645
646                                 rte_memcpy(rsa_result,
647                                                 cookie->output_array[0],
648                                                 alg_size_in_bytes);
649                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
650 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
651                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
652                                                 cookie->output_array[0],
653                                                 alg_size_in_bytes);
654 #endif
655                         }
656                 }
657         }
658         qat_clear_arrays_by_alg(cookie, xform->xform_type, alg_size_in_bytes,
659                         alg_size_in_bytes);
660 }
661
662 void
663 qat_asym_process_response(void **op, uint8_t *resp,
664                 void *op_cookie)
665 {
666         struct qat_asym_session *ctx;
667         struct icp_qat_fw_pke_resp *resp_msg =
668                         (struct icp_qat_fw_pke_resp *)resp;
669         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
670                         (resp_msg->opaque);
671         struct qat_asym_op_cookie *cookie = op_cookie;
672
673         if (cookie->error) {
674                 cookie->error = 0;
675                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
676                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
677                 QAT_DP_LOG(ERR, "Cookie status returned error");
678         } else {
679                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
680                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
681                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
682                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
683                         QAT_DP_LOG(ERR, "Asymmetric response status"
684                                         " returned error");
685                 }
686                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
687                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
688                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
689                         QAT_DP_LOG(ERR, "Asymmetric common status"
690                                         " returned error");
691                 }
692         }
693
694         if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
695                 ctx = (struct qat_asym_session *)get_asym_session_private_data(
696                         rx_op->asym->session, qat_asym_driver_id);
697                 qat_asym_collect_response(rx_op, cookie, ctx->xform);
698         } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
699                 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
700         }
701         *op = rx_op;
702
703 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
704         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
705                         sizeof(struct icp_qat_fw_pke_resp));
706 #endif
707 }
708
709 int
710 qat_asym_session_configure(struct rte_cryptodev *dev,
711                 struct rte_crypto_asym_xform *xform,
712                 struct rte_cryptodev_asym_session *sess,
713                 struct rte_mempool *mempool)
714 {
715         int err = 0;
716         void *sess_private_data;
717         struct qat_asym_session *session;
718
719         if (rte_mempool_get(mempool, &sess_private_data)) {
720                 QAT_LOG(ERR,
721                         "Couldn't get object from session mempool");
722                 return -ENOMEM;
723         }
724
725         session = sess_private_data;
726         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
727                 if (xform->modex.exponent.length == 0 ||
728                                 xform->modex.modulus.length == 0) {
729                         QAT_LOG(ERR, "Invalid mod exp input parameter");
730                         err = -EINVAL;
731                         goto error;
732                 }
733         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
734                 if (xform->modinv.modulus.length == 0) {
735                         QAT_LOG(ERR, "Invalid mod inv input parameter");
736                         err = -EINVAL;
737                         goto error;
738                 }
739         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
740                 if (xform->rsa.n.length == 0) {
741                         QAT_LOG(ERR, "Invalid rsa input parameter");
742                         err = -EINVAL;
743                         goto error;
744                 }
745         } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
746                         || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
747                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
748                 err = -EINVAL;
749                 goto error;
750         } else {
751                 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
752                 err = -EINVAL;
753                 goto error;
754         }
755
756         session->xform = xform;
757         qat_asym_build_req_tmpl(sess_private_data);
758         set_asym_session_private_data(sess, dev->driver_id,
759                 sess_private_data);
760
761         return 0;
762 error:
763         rte_mempool_put(mempool, sess_private_data);
764         return err;
765 }
766
767 unsigned int qat_asym_session_get_private_size(
768                 struct rte_cryptodev *dev __rte_unused)
769 {
770         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
771 }
772
773 void
774 qat_asym_session_clear(struct rte_cryptodev *dev,
775                 struct rte_cryptodev_asym_session *sess)
776 {
777         uint8_t index = dev->driver_id;
778         void *sess_priv = get_asym_session_private_data(sess, index);
779         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
780
781         if (sess_priv) {
782                 memset(s, 0, qat_asym_session_get_private_size(dev));
783                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
784
785                 set_asym_session_private_data(sess, index, NULL);
786                 rte_mempool_put(sess_mp, sess_priv);
787         }
788 }