drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include "qat_asym.h"
   8 #include "icp_qat_fw_pke.h"
   9 #include "icp_qat_fw.h"
  10 #include "qat_pke_functionality_arrays.h"
  11
  12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
  13
  14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
  15                 size_t arr_sz, size_t *size, uint32_t *func_id)
  16 {
  17         size_t i;
  18
  19         for (i = 0; i < arr_sz; i++) {
  20                 if (*size <= arr[i][0]) {
  21                         *size = arr[i][0];
  22                         *func_id = arr[i][1];
  23                         return 0;
  24                 }
  25         }
  26         return -1;
  27 }
  28
  29 static inline void qat_fill_req_tmpl(struct icp_qat_fw_pke_request *qat_req)
  30 {
  31         memset(qat_req, 0, sizeof(*qat_req));
  32         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
  33
  34         qat_req->pke_hdr.hdr_flags =
  35                         ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
  36                         (ICP_QAT_FW_COMN_REQ_FLAG_SET);
  37 }
  38
  39 static inline void qat_asym_build_req_tmpl(void *sess_private_data)
  40 {
  41         struct icp_qat_fw_pke_request *qat_req;
  42         struct qat_asym_session *session = sess_private_data;
  43
  44         qat_req = &session->req_tmpl;
  45         qat_fill_req_tmpl(qat_req);
  46 }
  47
  48 static size_t max_of(int n, ...)
  49 {
  50         va_list args;
  51         size_t len = 0, num;
  52         int i;
  53
  54         va_start(args, n);
  55         len = va_arg(args, size_t);
  56
  57         for (i = 0; i < n - 1; i++) {
  58                 num = va_arg(args, size_t);
  59                 if (num > len)
  60                         len = num;
  61         }
  62         va_end(args);
  63
  64         return len;
  65 }
  66
  67 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
  68                 int in_count, int out_count, int in_size, int out_size)
  69 {
  70         int i;
  71
  72         for (i = 0; i < in_count; i++)
  73                 memset(cookie->input_array[i], 0x0, in_size);
  74         for (i = 0; i < out_count; i++)
  75                 memset(cookie->output_array[i], 0x0, out_size);
  76 }
  77
  78 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
  79                 enum rte_crypto_asym_xform_type alg, int in_size, int out_size)
  80 {
  81         if (alg == RTE_CRYPTO_ASYM_XFORM_MODEX)
  82                 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
  83                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, in_size,
  84                                 out_size);
  85         else if (alg == RTE_CRYPTO_ASYM_XFORM_MODINV)
  86                 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
  87                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, in_size,
  88                                 out_size);
  89 }
  90
  91 static int qat_asym_check_nonzero(rte_crypto_param n)
  92 {
  93         if (n.length < 8) {
  94                 /* Not a case for any cryptograpic function except for DH
  95                  * generator which very often can be of one byte length
  96                  */
  97                 size_t i;
  98
  99                 if (n.data[n.length - 1] == 0x0) {
 100                         for (i = 0; i < n.length - 1; i++)
 101                                 if (n.data[i] != 0x0)
 102                                         break;
 103                         if (i == n.length - 1)
 104                                 return -(EINVAL);
 105                 }
 106         } else if (*(uint64_t *)&n.data[
 107                                 n.length - 8] == 0) {
 108                 /* Very likely it is zeroed modulus */
 109                 size_t i;
 110
 111                 for (i = 0; i < n.length - 8; i++)
 112                         if (n.data[i] != 0x0)
 113                                 break;
 114                 if (i == n.length - 8)
 115                         return -(EINVAL);
 116         }
 117
 118         return 0;
 119 }
 120
 121 static int
 122 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 123                 struct icp_qat_fw_pke_request *qat_req,
 124                 struct qat_asym_op_cookie *cookie,
 125                 struct rte_crypto_asym_xform *xform)
 126 {
 127         int err = 0;
 128         size_t alg_size;
 129         size_t alg_size_in_bytes;
 130         uint32_t func_id = 0;
 131
 132         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 133                 err = qat_asym_check_nonzero(xform->modex.modulus);
 134                 if (err) {
 135                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
 136                                         " aborting this operation");
 137                         return err;
 138                 }
 139
 140                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
 141                                xform->modex.exponent.length,
 142                                xform->modex.modulus.length);
 143                 alg_size = alg_size_in_bytes << 3;
 144
 145                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
 146                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
 147                                 &alg_size, &func_id)) {
 148                         return -(EINVAL);
 149                 }
 150
 151                 alg_size_in_bytes = alg_size >> 3;
 152                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 153                         asym_op->modex.base.length
 154                         , asym_op->modex.base.data,
 155                         asym_op->modex.base.length);
 156                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 157                         xform->modex.exponent.length
 158                         , xform->modex.exponent.data,
 159                         xform->modex.exponent.length);
 160                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
 161                         xform->modex.modulus.length,
 162                         xform->modex.modulus.data,
 163                         xform->modex.modulus.length);
 164                 cookie->alg_size = alg_size;
 165                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 166                 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
 167                 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
 168 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 169                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
 170                                 cookie->input_array[0],
 171                                 alg_size_in_bytes);
 172                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
 173                                 cookie->input_array[1],
 174                                 alg_size_in_bytes);
 175                 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
 176                                 cookie->input_array[2],
 177                                 alg_size_in_bytes);
 178 #endif
 179         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 180                 err = qat_asym_check_nonzero(xform->modinv.modulus);
 181                 if (err) {
 182                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
 183                                         " inverse, aborting this operation");
 184                         return err;
 185                 }
 186
 187                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
 188                                 xform->modinv.modulus.length);
 189                 alg_size = alg_size_in_bytes << 3;
 190
 191                 if (xform->modinv.modulus.data[
 192                                 xform->modinv.modulus.length - 1] & 0x01) {
 193                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
 194                                         sizeof(MOD_INV_IDS_ODD)/
 195                                         sizeof(*MOD_INV_IDS_ODD),
 196                                         &alg_size, &func_id)) {
 197                                 return -(EINVAL);
 198                         }
 199                 } else {
 200                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
 201                                         sizeof(MOD_INV_IDS_EVEN)/
 202                                         sizeof(*MOD_INV_IDS_EVEN),
 203                                         &alg_size, &func_id)) {
 204                                 return -(EINVAL);
 205                         }
 206                 }
 207
 208                 alg_size_in_bytes = alg_size >> 3;
 209                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 210                         asym_op->modinv.base.length
 211                                 , asym_op->modinv.base.data,
 212                                 asym_op->modinv.base.length);
 213                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 214                                 xform->modinv.modulus.length
 215                                 , xform->modinv.modulus.data,
 216                                 xform->modinv.modulus.length);
 217                 cookie->alg_size = alg_size;
 218                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 219                 qat_req->input_param_count =
 220                                 QAT_ASYM_MODINV_NUM_IN_PARAMS;
 221                 qat_req->output_param_count =
 222                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
 223 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 224                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
 225                                 cookie->input_array[0],
 226                                 alg_size_in_bytes);
 227                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
 228                                 cookie->input_array[1],
 229                                 alg_size_in_bytes);
 230 #endif
 231         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 232                 err = qat_asym_check_nonzero(xform->rsa.n);
 233                 if (err) {
 234                         QAT_LOG(ERR, "Empty modulus in RSA"
 235                                         " inverse, aborting this operation");
 236                         return err;
 237                 }
 238
 239                 alg_size_in_bytes = xform->rsa.n.length;
 240                 alg_size = alg_size_in_bytes << 3;
 241
 242                 qat_req->input_param_count =
 243                                 QAT_ASYM_RSA_NUM_IN_PARAMS;
 244                 qat_req->output_param_count =
 245                                 QAT_ASYM_RSA_NUM_OUT_PARAMS;
 246
 247                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 248                                 asym_op->rsa.op_type ==
 249                                                 RTE_CRYPTO_ASYM_OP_VERIFY) {
 250
 251                         if (qat_asym_get_sz_and_func_id(RSA_ENC_IDS,
 252                                         sizeof(RSA_ENC_IDS)/
 253                                         sizeof(*RSA_ENC_IDS),
 254                                         &alg_size, &func_id)) {
 255                                 err = -(EINVAL);
 256                                 QAT_LOG(ERR,
 257                                         "Not supported RSA parameter size (key)");
 258                                 return err;
 259                         }
 260                         alg_size_in_bytes = alg_size >> 3;
 261                         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 262                                 switch (asym_op->rsa.pad) {
 263                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 264                                         rte_memcpy(cookie->input_array[0] +
 265                                                 alg_size_in_bytes -
 266                                                 asym_op->rsa.message.length
 267                                                 , asym_op->rsa.message.data,
 268                                                 asym_op->rsa.message.length);
 269                                         break;
 270                                 default:
 271                                         err = -(EINVAL);
 272                                         QAT_LOG(ERR,
 273                                                 "Invalid RSA padding (Encryption)");
 274                                         return err;
 275                                 }
 276 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 277                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Message",
 278                                                 cookie->input_array[0],
 279                                                 alg_size_in_bytes);
 280 #endif
 281                         } else {
 282                                 switch (asym_op->rsa.pad) {
 283                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 284                                         rte_memcpy(cookie->input_array[0],
 285                                                 asym_op->rsa.sign.data,
 286                                                 alg_size_in_bytes);
 287                                         break;
 288                                 default:
 289                                         err = -(EINVAL);
 290                                         QAT_LOG(ERR,
 291                                                 "Invalid RSA padding (Verify)");
 292                                         return err;
 293                                 }
 294
 295 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 296                                 QAT_DP_HEXDUMP_LOG(DEBUG, " RSA Signature",
 297                                                 cookie->input_array[0],
 298                                                 alg_size_in_bytes);
 299 #endif
 300
 301                         }
 302                         rte_memcpy(cookie->input_array[1] +
 303                                         alg_size_in_bytes -
 304                                         xform->rsa.e.length
 305                                         , xform->rsa.e.data,
 306                                         xform->rsa.e.length);
 307                         rte_memcpy(cookie->input_array[2] +
 308                                         alg_size_in_bytes -
 309                                         xform->rsa.n.length,
 310                                         xform->rsa.n.data,
 311                                         xform->rsa.n.length);
 312
 313                         cookie->alg_size = alg_size;
 314                         qat_req->pke_hdr.cd_pars.func_id = func_id;
 315
 316 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 317                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Public Key",
 318                                         cookie->input_array[1], alg_size_in_bytes);
 319                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Modulus",
 320                                         cookie->input_array[2], alg_size_in_bytes);
 321 #endif
 322                 } else {
 323                         if (asym_op->rsa.op_type ==
 324                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 325                                 switch (asym_op->rsa.pad) {
 326                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 327                                         rte_memcpy(cookie->input_array[0]
 328                                                 + alg_size_in_bytes -
 329                                                 asym_op->rsa.cipher.length,
 330                                                 asym_op->rsa.cipher.data,
 331                                                 asym_op->rsa.cipher.length);
 332                                         break;
 333                                 default:
 334                                         QAT_LOG(ERR,
 335                                                 "Invalid padding of RSA (Decrypt)");
 336                                         return -(EINVAL);
 337                                 }
 338
 339                         } else if (asym_op->rsa.op_type ==
 340                                         RTE_CRYPTO_ASYM_OP_SIGN) {
 341                                 switch (asym_op->rsa.pad) {
 342                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 343                                         rte_memcpy(cookie->input_array[0]
 344                                                 + alg_size_in_bytes -
 345                                                 asym_op->rsa.message.length,
 346                                                 asym_op->rsa.message.data,
 347                                                 asym_op->rsa.message.length);
 348                                         break;
 349                                 default:
 350                                         QAT_LOG(ERR,
 351                                                 "Invalid padding of RSA (Signature)");
 352                                         return -(EINVAL);
 353                                 }
 354                         }
 355                         if (xform->rsa.key_type == RTE_RSA_KET_TYPE_QT) {
 356
 357                                 qat_req->input_param_count =
 358                                                 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
 359                                 if (qat_asym_get_sz_and_func_id(RSA_DEC_CRT_IDS,
 360                                                 sizeof(RSA_DEC_CRT_IDS)/
 361                                                 sizeof(*RSA_DEC_CRT_IDS),
 362                                                 &alg_size, &func_id)) {
 363                                         return -(EINVAL);
 364                                 }
 365                                 alg_size_in_bytes = alg_size >> 3;
 366
 367                                 rte_memcpy(cookie->input_array[1] +
 368                                                 (alg_size_in_bytes >> 1) -
 369                                                 xform->rsa.qt.p.length
 370                                                 , xform->rsa.qt.p.data,
 371                                                 xform->rsa.qt.p.length);
 372                                 rte_memcpy(cookie->input_array[2] +
 373                                                 (alg_size_in_bytes >> 1) -
 374                                                 xform->rsa.qt.q.length
 375                                                 , xform->rsa.qt.q.data,
 376                                                 xform->rsa.qt.q.length);
 377                                 rte_memcpy(cookie->input_array[3] +
 378                                                 (alg_size_in_bytes >> 1) -
 379                                                 xform->rsa.qt.dP.length
 380                                                 , xform->rsa.qt.dP.data,
 381                                                 xform->rsa.qt.dP.length);
 382                                 rte_memcpy(cookie->input_array[4] +
 383                                                 (alg_size_in_bytes >> 1) -
 384                                                 xform->rsa.qt.dQ.length
 385                                                 , xform->rsa.qt.dQ.data,
 386                                                 xform->rsa.qt.dQ.length);
 387                                 rte_memcpy(cookie->input_array[5] +
 388                                                 (alg_size_in_bytes >> 1) -
 389                                                 xform->rsa.qt.qInv.length
 390                                                 , xform->rsa.qt.qInv.data,
 391                                                 xform->rsa.qt.qInv.length);
 392                                 cookie->alg_size = alg_size;
 393                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 394
 395 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 396                                 QAT_DP_HEXDUMP_LOG(DEBUG, "C",
 397                                                 cookie->input_array[0],
 398                                                 alg_size_in_bytes);
 399                                 QAT_DP_HEXDUMP_LOG(DEBUG, "p",
 400                                                 cookie->input_array[1],
 401                                                 alg_size_in_bytes);
 402                                 QAT_DP_HEXDUMP_LOG(DEBUG, "q",
 403                                                 cookie->input_array[2],
 404                                                 alg_size_in_bytes);
 405                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 406                                                 "dP", cookie->input_array[3],
 407                                                 alg_size_in_bytes);
 408                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 409                                                 "dQ", cookie->input_array[4],
 410                                                 alg_size_in_bytes);
 411                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 412                                                 "qInv", cookie->input_array[5],
 413                                                 alg_size_in_bytes);
 414 #endif
 415                         } else if (xform->rsa.key_type ==
 416                                         RTE_RSA_KEY_TYPE_EXP) {
 417                                 if (qat_asym_get_sz_and_func_id(
 418                                                 RSA_DEC_IDS,
 419                                                 sizeof(RSA_DEC_IDS)/
 420                                                 sizeof(*RSA_DEC_IDS),
 421                                                 &alg_size, &func_id)) {
 422                                         return -(EINVAL);
 423                                 }
 424                                 alg_size_in_bytes = alg_size >> 3;
 425                                 rte_memcpy(cookie->input_array[1] +
 426                                                 alg_size_in_bytes -
 427                                                 xform->rsa.d.length,
 428                                                 xform->rsa.d.data,
 429                                                 xform->rsa.d.length);
 430                                 rte_memcpy(cookie->input_array[2] +
 431                                                 alg_size_in_bytes -
 432                                                 xform->rsa.n.length,
 433                                                 xform->rsa.n.data,
 434                                                 xform->rsa.n.length);
 435 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 436                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA ciphertext",
 437                                         cookie->input_array[0],
 438                                         alg_size_in_bytes);
 439                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA d", cookie->input_array[1],
 440                                         alg_size_in_bytes);
 441                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA n", cookie->input_array[2],
 442                                         alg_size_in_bytes);
 443 #endif
 444
 445                                 cookie->alg_size = alg_size;
 446                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 447                         } else {
 448                                 QAT_LOG(ERR, "Invalid RSA key type");
 449                                 return -(EINVAL);
 450                         }
 451                 }
 452         } else {
 453                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 454                 return -(EINVAL);
 455         }
 456         return 0;
 457 }
 458
 459 int
 460 qat_asym_build_request(void *in_op,
 461                         uint8_t *out_msg,
 462                         void *op_cookie,
 463                         __rte_unused enum qat_device_gen qat_dev_gen)
 464 {
 465         struct qat_asym_session *ctx;
 466         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 467         struct rte_crypto_asym_op *asym_op = op->asym;
 468         struct icp_qat_fw_pke_request *qat_req =
 469                         (struct icp_qat_fw_pke_request *)out_msg;
 470         struct qat_asym_op_cookie *cookie =
 471                                 (struct qat_asym_op_cookie *)op_cookie;
 472         int err = 0;
 473
 474         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 475         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 476                 ctx = (struct qat_asym_session *)
 477                         get_asym_session_private_data(
 478                         op->asym->session, qat_asym_driver_id);
 479                 if (unlikely(ctx == NULL)) {
 480                         QAT_LOG(ERR, "Session has not been created for this device");
 481                         goto error;
 482                 }
 483                 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
 484                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
 485                 if (err) {
 486                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 487                         goto error;
 488                 }
 489         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 490                 qat_fill_req_tmpl(qat_req);
 491                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
 492                                 op->asym->xform);
 493                 if (err) {
 494                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 495                         goto error;
 496                 }
 497         } else {
 498                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
 499                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 500                 goto error;
 501         }
 502
 503         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 504         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 505         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 506
 507 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 508         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 509                         sizeof(struct icp_qat_fw_pke_request));
 510 #endif
 511
 512         return 0;
 513 error:
 514
 515         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 516
 517 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 518         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 519                 sizeof(struct icp_qat_fw_pke_request));
 520 #endif
 521
 522         qat_req->output_param_count = 0;
 523         qat_req->input_param_count = 0;
 524         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 525         cookie->error |= err;
 526
 527         return 0;
 528 }
 529
 530 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
 531                 struct qat_asym_op_cookie *cookie,
 532                 struct rte_crypto_asym_xform *xform)
 533 {
 534         size_t alg_size, alg_size_in_bytes = 0;
 535         struct rte_crypto_asym_op *asym_op = rx_op->asym;
 536
 537         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 538                 rte_crypto_param n = xform->modex.modulus;
 539
 540                 alg_size = cookie->alg_size;
 541                 alg_size_in_bytes = alg_size >> 3;
 542                 uint8_t *modexp_result = asym_op->modex.result.data;
 543
 544                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 545                         rte_memcpy(modexp_result +
 546                                 (asym_op->modex.result.length -
 547                                         n.length),
 548                                 cookie->output_array[0] + alg_size_in_bytes
 549                                 - n.length, n.length
 550                                 );
 551                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 552 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 553                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
 554                                         cookie->output_array[0],
 555                                         alg_size_in_bytes);
 556
 557 #endif
 558                 }
 559         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 560                 rte_crypto_param n = xform->modinv.modulus;
 561
 562                 alg_size = cookie->alg_size;
 563                 alg_size_in_bytes = alg_size >> 3;
 564                 uint8_t *modinv_result = asym_op->modinv.result.data;
 565
 566                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 567                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 568                                 - n.length),
 569                                 cookie->output_array[0] + alg_size_in_bytes
 570                                 - n.length, n.length);
 571                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 572 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 573                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
 574                                         cookie->output_array[0],
 575                                         alg_size_in_bytes);
 576 #endif
 577                 }
 578         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 579
 580                 alg_size = cookie->alg_size;
 581                 alg_size_in_bytes = alg_size >> 3;
 582                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 583                                 asym_op->rsa.op_type ==
 584                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 585                         if (asym_op->rsa.op_type ==
 586                                         RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 587                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 588
 589                                 rte_memcpy(rsa_result,
 590                                                 cookie->output_array[0],
 591                                                 alg_size_in_bytes);
 592                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 593 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 594                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Encrypted data",
 595                                                 cookie->output_array[0],
 596                                                 alg_size_in_bytes);
 597 #endif
 598                         } else if (asym_op->rsa.op_type ==
 599                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 600                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 601
 602                                 switch (asym_op->rsa.pad) {
 603                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 604                                         rte_memcpy(rsa_result,
 605                                                         cookie->output_array[0],
 606                                                         alg_size_in_bytes);
 607                                         rx_op->status =
 608                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 609 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 610                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 611                                                 cookie->output_array[0],
 612                                                 alg_size_in_bytes);
 613 #endif
 614                                         break;
 615                                 default:
 616                                         QAT_LOG(ERR, "Padding not supported");
 617                                         rx_op->status =
 618                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 619                                         break;
 620                                 }
 621                         }
 622                 } else {
 623                         if (asym_op->rsa.op_type ==
 624                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 625                                 uint8_t *rsa_result = asym_op->rsa.message.data;
 626
 627                                 switch (asym_op->rsa.pad) {
 628                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 629                                         rte_memcpy(rsa_result,
 630                                                 cookie->output_array[0],
 631                                                 alg_size_in_bytes);
 632                                         break;
 633                                 default:
 634                                         QAT_LOG(ERR, "Padding not supported");
 635                                         rx_op->status =
 636                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 637                                         break;
 638                                 }
 639 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 640                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Decrypted Message",
 641                                                 rsa_result, alg_size_in_bytes);
 642 #endif
 643                         } else if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
 644                                 uint8_t *rsa_result = asym_op->rsa.sign.data;
 645
 646                                 rte_memcpy(rsa_result,
 647                                                 cookie->output_array[0],
 648                                                 alg_size_in_bytes);
 649                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 650 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 651                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 652                                                 cookie->output_array[0],
 653                                                 alg_size_in_bytes);
 654 #endif
 655                         }
 656                 }
 657         }
 658         qat_clear_arrays_by_alg(cookie, xform->xform_type, alg_size_in_bytes,
 659                         alg_size_in_bytes);
 660 }
 661
 662 void
 663 qat_asym_process_response(void **op, uint8_t *resp,
 664                 void *op_cookie)
 665 {
 666         struct qat_asym_session *ctx;
 667         struct icp_qat_fw_pke_resp *resp_msg =
 668                         (struct icp_qat_fw_pke_resp *)resp;
 669         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
 670                         (resp_msg->opaque);
 671         struct qat_asym_op_cookie *cookie = op_cookie;
 672
 673         if (cookie->error) {
 674                 cookie->error = 0;
 675                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 676                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 677                 QAT_DP_LOG(ERR, "Cookie status returned error");
 678         } else {
 679                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 680                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 681                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 682                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 683                         QAT_DP_LOG(ERR, "Asymmetric response status"
 684                                         " returned error");
 685                 }
 686                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 687                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 688                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 689                         QAT_DP_LOG(ERR, "Asymmetric common status"
 690                                         " returned error");
 691                 }
 692         }
 693
 694         if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 695                 ctx = (struct qat_asym_session *)get_asym_session_private_data(
 696                         rx_op->asym->session, qat_asym_driver_id);
 697                 qat_asym_collect_response(rx_op, cookie, ctx->xform);
 698         } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 699                 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
 700         }
 701         *op = rx_op;
 702
 703 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 704         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
 705                         sizeof(struct icp_qat_fw_pke_resp));
 706 #endif
 707 }
 708
 709 int
 710 qat_asym_session_configure(struct rte_cryptodev *dev,
 711                 struct rte_crypto_asym_xform *xform,
 712                 struct rte_cryptodev_asym_session *sess,
 713                 struct rte_mempool *mempool)
 714 {
 715         int err = 0;
 716         void *sess_private_data;
 717         struct qat_asym_session *session;
 718
 719         if (rte_mempool_get(mempool, &sess_private_data)) {
 720                 QAT_LOG(ERR,
 721                         "Couldn't get object from session mempool");
 722                 return -ENOMEM;
 723         }
 724
 725         session = sess_private_data;
 726         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 727                 if (xform->modex.exponent.length == 0 ||
 728                                 xform->modex.modulus.length == 0) {
 729                         QAT_LOG(ERR, "Invalid mod exp input parameter");
 730                         err = -EINVAL;
 731                         goto error;
 732                 }
 733         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 734                 if (xform->modinv.modulus.length == 0) {
 735                         QAT_LOG(ERR, "Invalid mod inv input parameter");
 736                         err = -EINVAL;
 737                         goto error;
 738                 }
 739         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 740                 if (xform->rsa.n.length == 0) {
 741                         QAT_LOG(ERR, "Invalid rsa input parameter");
 742                         err = -EINVAL;
 743                         goto error;
 744                 }
 745         } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
 746                         || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
 747                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 748                 err = -EINVAL;
 749                 goto error;
 750         } else {
 751                 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
 752                 err = -EINVAL;
 753                 goto error;
 754         }
 755
 756         session->xform = xform;
 757         qat_asym_build_req_tmpl(sess_private_data);
 758         set_asym_session_private_data(sess, dev->driver_id,
 759                 sess_private_data);
 760
 761         return 0;
 762 error:
 763         rte_mempool_put(mempool, sess_private_data);
 764         return err;
 765 }
 766
 767 unsigned int qat_asym_session_get_private_size(
 768                 struct rte_cryptodev *dev __rte_unused)
 769 {
 770         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
 771 }
 772
 773 void
 774 qat_asym_session_clear(struct rte_cryptodev *dev,
 775                 struct rte_cryptodev_asym_session *sess)
 776 {
 777         uint8_t index = dev->driver_id;
 778         void *sess_priv = get_asym_session_private_data(sess, index);
 779         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
 780
 781         if (sess_priv) {
 782                 memset(s, 0, qat_asym_session_get_private_size(dev));
 783                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 784
 785                 set_asym_session_private_data(sess, index, NULL);
 786                 rte_mempool_put(sess_mp, sess_priv);
 787         }
 788 }