drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 - 2022 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include <cryptodev_pmd.h>
   8
   9 #include "icp_qat_fw_pke.h"
  10 #include "icp_qat_fw.h"
  11 #include "qat_pke_functionality_arrays.h"
  12
  13 #include "qat_device.h"
  14
  15 #include "qat_logs.h"
  16 #include "qat_asym.h"
  17
  18 uint8_t qat_asym_driver_id;
  19
  20 struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[QAT_N_GENS];
  21
  22 /* An rte_driver is needed in the registration of both the device and the driver
  23  * with cryptodev.
  24  * The actual qat pci's rte_driver can't be used as its name represents
  25  * the whole pci device with all services. Think of this as a holder for a name
  26  * for the crypto part of the pci device.
  27  */
  28 static const char qat_asym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_ASYM_PMD);
  29 static const struct rte_driver cryptodev_qat_asym_driver = {
  30         .name = qat_asym_drv_name,
  31         .alias = qat_asym_drv_name
  32 };
  33
  34
  35 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
  36                 int in_count, int out_count, int alg_size)
  37 {
  38         int i;
  39
  40         for (i = 0; i < in_count; i++)
  41                 memset(cookie->input_array[i], 0x0, alg_size);
  42         for (i = 0; i < out_count; i++)
  43                 memset(cookie->output_array[i], 0x0, alg_size);
  44 }
  45
  46 static void qat_clear_arrays_crt(struct qat_asym_op_cookie *cookie,
  47                 int alg_size)
  48 {
  49         int i;
  50
  51         memset(cookie->input_array[0], 0x0, alg_size);
  52         for (i = 1; i < QAT_ASYM_RSA_QT_NUM_IN_PARAMS; i++)
  53                 memset(cookie->input_array[i], 0x0, alg_size / 2);
  54         for (i = 0; i < QAT_ASYM_RSA_NUM_OUT_PARAMS; i++)
  55                 memset(cookie->output_array[i], 0x0, alg_size);
  56 }
  57
  58 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
  59                 struct rte_crypto_asym_xform *xform, int alg_size)
  60 {
  61         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX)
  62                 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
  63                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, alg_size);
  64         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV)
  65                 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
  66                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, alg_size);
  67         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
  68                 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT)
  69                         qat_clear_arrays_crt(cookie, alg_size);
  70                 else {
  71                         qat_clear_arrays(cookie, QAT_ASYM_RSA_NUM_IN_PARAMS,
  72                                 QAT_ASYM_RSA_NUM_OUT_PARAMS, alg_size);
  73                 }
  74         }
  75 }
  76
  77 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
  78
  79 static int
  80 qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
  81                 size_t arr_sz, size_t *size, uint32_t *func_id)
  82 {
  83         size_t i;
  84
  85         for (i = 0; i < arr_sz; i++) {
  86                 if (*size <= arr[i][0]) {
  87                         *size = arr[i][0];
  88                         *func_id = arr[i][1];
  89                         return 0;
  90                 }
  91         }
  92         return -1;
  93 }
  94
  95 static size_t
  96 max_of(int n, ...)
  97 {
  98         va_list args;
  99         size_t len = 0, num;
 100         int i;
 101
 102         va_start(args, n);
 103         len = va_arg(args, size_t);
 104
 105         for (i = 0; i < n - 1; i++) {
 106                 num = va_arg(args, size_t);
 107                 if (num > len)
 108                         len = num;
 109         }
 110         va_end(args);
 111
 112         return len;
 113 }
 114
 115 static int
 116 qat_asym_check_nonzero(rte_crypto_param n)
 117 {
 118         if (n.length < 8) {
 119                 /* Not a case for any cryptographic function except for DH
 120                  * generator which very often can be of one byte length
 121                  */
 122                 size_t i;
 123
 124                 if (n.data[n.length - 1] == 0x0) {
 125                         for (i = 0; i < n.length - 1; i++)
 126                                 if (n.data[i] != 0x0)
 127                                         break;
 128                         if (i == n.length - 1)
 129                                 return -(EINVAL);
 130                 }
 131         } else if (*(uint64_t *)&n.data[
 132                                 n.length - 8] == 0) {
 133                 /* Very likely it is zeroed modulus */
 134                 size_t i;
 135
 136                 for (i = 0; i < n.length - 8; i++)
 137                         if (n.data[i] != 0x0)
 138                                 break;
 139                 if (i == n.length - 8)
 140                         return -(EINVAL);
 141         }
 142
 143         return 0;
 144 }
 145
 146 static int
 147 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 148                 struct icp_qat_fw_pke_request *qat_req,
 149                 struct qat_asym_op_cookie *cookie,
 150                 struct rte_crypto_asym_xform *xform)
 151 {
 152         int err = 0;
 153         size_t alg_size;
 154         size_t alg_size_in_bytes;
 155         uint32_t func_id = 0;
 156
 157         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 158                 err = qat_asym_check_nonzero(xform->modex.modulus);
 159                 if (err) {
 160                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
 161                                         " aborting this operation");
 162                         return err;
 163                 }
 164
 165                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
 166                                xform->modex.exponent.length,
 167                                xform->modex.modulus.length);
 168                 alg_size = alg_size_in_bytes << 3;
 169
 170                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
 171                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
 172                                 &alg_size, &func_id)) {
 173                         return -(EINVAL);
 174                 }
 175
 176                 alg_size_in_bytes = alg_size >> 3;
 177                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 178                         asym_op->modex.base.length
 179                         , asym_op->modex.base.data,
 180                         asym_op->modex.base.length);
 181                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 182                         xform->modex.exponent.length
 183                         , xform->modex.exponent.data,
 184                         xform->modex.exponent.length);
 185                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
 186                         xform->modex.modulus.length,
 187                         xform->modex.modulus.data,
 188                         xform->modex.modulus.length);
 189                 cookie->alg_size = alg_size;
 190                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 191                 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
 192                 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
 193 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 194                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
 195                                 cookie->input_array[0],
 196                                 alg_size_in_bytes);
 197                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
 198                                 cookie->input_array[1],
 199                                 alg_size_in_bytes);
 200                 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
 201                                 cookie->input_array[2],
 202                                 alg_size_in_bytes);
 203 #endif
 204         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 205                 err = qat_asym_check_nonzero(xform->modinv.modulus);
 206                 if (err) {
 207                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
 208                                         " inverse, aborting this operation");
 209                         return err;
 210                 }
 211
 212                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
 213                                 xform->modinv.modulus.length);
 214                 alg_size = alg_size_in_bytes << 3;
 215
 216                 if (xform->modinv.modulus.data[
 217                                 xform->modinv.modulus.length - 1] & 0x01) {
 218                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
 219                                         sizeof(MOD_INV_IDS_ODD)/
 220                                         sizeof(*MOD_INV_IDS_ODD),
 221                                         &alg_size, &func_id)) {
 222                                 return -(EINVAL);
 223                         }
 224                 } else {
 225                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
 226                                         sizeof(MOD_INV_IDS_EVEN)/
 227                                         sizeof(*MOD_INV_IDS_EVEN),
 228                                         &alg_size, &func_id)) {
 229                                 return -(EINVAL);
 230                         }
 231                 }
 232
 233                 alg_size_in_bytes = alg_size >> 3;
 234                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 235                         asym_op->modinv.base.length
 236                                 , asym_op->modinv.base.data,
 237                                 asym_op->modinv.base.length);
 238                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 239                                 xform->modinv.modulus.length
 240                                 , xform->modinv.modulus.data,
 241                                 xform->modinv.modulus.length);
 242                 cookie->alg_size = alg_size;
 243                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 244                 qat_req->input_param_count =
 245                                 QAT_ASYM_MODINV_NUM_IN_PARAMS;
 246                 qat_req->output_param_count =
 247                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
 248 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 249                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
 250                                 cookie->input_array[0],
 251                                 alg_size_in_bytes);
 252                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
 253                                 cookie->input_array[1],
 254                                 alg_size_in_bytes);
 255 #endif
 256         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 257                 err = qat_asym_check_nonzero(xform->rsa.n);
 258                 if (err) {
 259                         QAT_LOG(ERR, "Empty modulus in RSA"
 260                                         " inverse, aborting this operation");
 261                         return err;
 262                 }
 263
 264                 alg_size_in_bytes = xform->rsa.n.length;
 265                 alg_size = alg_size_in_bytes << 3;
 266
 267                 qat_req->input_param_count =
 268                                 QAT_ASYM_RSA_NUM_IN_PARAMS;
 269                 qat_req->output_param_count =
 270                                 QAT_ASYM_RSA_NUM_OUT_PARAMS;
 271
 272                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 273                                 asym_op->rsa.op_type ==
 274                                                 RTE_CRYPTO_ASYM_OP_VERIFY) {
 275
 276                         if (qat_asym_get_sz_and_func_id(RSA_ENC_IDS,
 277                                         sizeof(RSA_ENC_IDS)/
 278                                         sizeof(*RSA_ENC_IDS),
 279                                         &alg_size, &func_id)) {
 280                                 err = -(EINVAL);
 281                                 QAT_LOG(ERR,
 282                                         "Not supported RSA parameter size (key)");
 283                                 return err;
 284                         }
 285                         alg_size_in_bytes = alg_size >> 3;
 286                         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 287                                 switch (asym_op->rsa.pad) {
 288                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 289                                         rte_memcpy(cookie->input_array[0] +
 290                                                 alg_size_in_bytes -
 291                                                 asym_op->rsa.message.length
 292                                                 , asym_op->rsa.message.data,
 293                                                 asym_op->rsa.message.length);
 294                                         break;
 295                                 default:
 296                                         err = -(EINVAL);
 297                                         QAT_LOG(ERR,
 298                                                 "Invalid RSA padding (Encryption)");
 299                                         return err;
 300                                 }
 301 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 302                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Message",
 303                                                 cookie->input_array[0],
 304                                                 alg_size_in_bytes);
 305 #endif
 306                         } else {
 307                                 switch (asym_op->rsa.pad) {
 308                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 309                                         rte_memcpy(cookie->input_array[0],
 310                                                 asym_op->rsa.sign.data,
 311                                                 alg_size_in_bytes);
 312                                         break;
 313                                 default:
 314                                         err = -(EINVAL);
 315                                         QAT_LOG(ERR,
 316                                                 "Invalid RSA padding (Verify)");
 317                                         return err;
 318                                 }
 319
 320 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 321                                 QAT_DP_HEXDUMP_LOG(DEBUG, " RSA Signature",
 322                                                 cookie->input_array[0],
 323                                                 alg_size_in_bytes);
 324 #endif
 325
 326                         }
 327                         rte_memcpy(cookie->input_array[1] +
 328                                         alg_size_in_bytes -
 329                                         xform->rsa.e.length
 330                                         , xform->rsa.e.data,
 331                                         xform->rsa.e.length);
 332                         rte_memcpy(cookie->input_array[2] +
 333                                         alg_size_in_bytes -
 334                                         xform->rsa.n.length,
 335                                         xform->rsa.n.data,
 336                                         xform->rsa.n.length);
 337
 338                         cookie->alg_size = alg_size;
 339                         qat_req->pke_hdr.cd_pars.func_id = func_id;
 340
 341 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 342                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Public Key",
 343                                         cookie->input_array[1], alg_size_in_bytes);
 344                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Modulus",
 345                                         cookie->input_array[2], alg_size_in_bytes);
 346 #endif
 347                 } else {
 348                         if (asym_op->rsa.op_type ==
 349                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 350                                 switch (asym_op->rsa.pad) {
 351                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 352                                         rte_memcpy(cookie->input_array[0]
 353                                                 + alg_size_in_bytes -
 354                                                 asym_op->rsa.cipher.length,
 355                                                 asym_op->rsa.cipher.data,
 356                                                 asym_op->rsa.cipher.length);
 357                                         break;
 358                                 default:
 359                                         QAT_LOG(ERR,
 360                                                 "Invalid padding of RSA (Decrypt)");
 361                                         return -(EINVAL);
 362                                 }
 363
 364                         } else if (asym_op->rsa.op_type ==
 365                                         RTE_CRYPTO_ASYM_OP_SIGN) {
 366                                 switch (asym_op->rsa.pad) {
 367                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 368                                         rte_memcpy(cookie->input_array[0]
 369                                                 + alg_size_in_bytes -
 370                                                 asym_op->rsa.message.length,
 371                                                 asym_op->rsa.message.data,
 372                                                 asym_op->rsa.message.length);
 373                                         break;
 374                                 default:
 375                                         QAT_LOG(ERR,
 376                                                 "Invalid padding of RSA (Signature)");
 377                                         return -(EINVAL);
 378                                 }
 379                         }
 380                         if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT) {
 381
 382                                 qat_req->input_param_count =
 383                                                 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
 384                                 if (qat_asym_get_sz_and_func_id(RSA_DEC_CRT_IDS,
 385                                                 sizeof(RSA_DEC_CRT_IDS)/
 386                                                 sizeof(*RSA_DEC_CRT_IDS),
 387                                                 &alg_size, &func_id)) {
 388                                         return -(EINVAL);
 389                                 }
 390                                 alg_size_in_bytes = alg_size >> 3;
 391
 392                                 rte_memcpy(cookie->input_array[1] +
 393                                                 (alg_size_in_bytes >> 1) -
 394                                                 xform->rsa.qt.p.length
 395                                                 , xform->rsa.qt.p.data,
 396                                                 xform->rsa.qt.p.length);
 397                                 rte_memcpy(cookie->input_array[2] +
 398                                                 (alg_size_in_bytes >> 1) -
 399                                                 xform->rsa.qt.q.length
 400                                                 , xform->rsa.qt.q.data,
 401                                                 xform->rsa.qt.q.length);
 402                                 rte_memcpy(cookie->input_array[3] +
 403                                                 (alg_size_in_bytes >> 1) -
 404                                                 xform->rsa.qt.dP.length
 405                                                 , xform->rsa.qt.dP.data,
 406                                                 xform->rsa.qt.dP.length);
 407                                 rte_memcpy(cookie->input_array[4] +
 408                                                 (alg_size_in_bytes >> 1) -
 409                                                 xform->rsa.qt.dQ.length
 410                                                 , xform->rsa.qt.dQ.data,
 411                                                 xform->rsa.qt.dQ.length);
 412                                 rte_memcpy(cookie->input_array[5] +
 413                                                 (alg_size_in_bytes >> 1) -
 414                                                 xform->rsa.qt.qInv.length
 415                                                 , xform->rsa.qt.qInv.data,
 416                                                 xform->rsa.qt.qInv.length);
 417                                 cookie->alg_size = alg_size;
 418                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 419
 420 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 421                                 QAT_DP_HEXDUMP_LOG(DEBUG, "C",
 422                                                 cookie->input_array[0],
 423                                                 alg_size_in_bytes);
 424                                 QAT_DP_HEXDUMP_LOG(DEBUG, "p",
 425                                                 cookie->input_array[1],
 426                                                 alg_size_in_bytes);
 427                                 QAT_DP_HEXDUMP_LOG(DEBUG, "q",
 428                                                 cookie->input_array[2],
 429                                                 alg_size_in_bytes);
 430                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 431                                                 "dP", cookie->input_array[3],
 432                                                 alg_size_in_bytes);
 433                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 434                                                 "dQ", cookie->input_array[4],
 435                                                 alg_size_in_bytes);
 436                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 437                                                 "qInv", cookie->input_array[5],
 438                                                 alg_size_in_bytes);
 439 #endif
 440                         } else if (xform->rsa.key_type ==
 441                                         RTE_RSA_KEY_TYPE_EXP) {
 442                                 if (qat_asym_get_sz_and_func_id(
 443                                                 RSA_DEC_IDS,
 444                                                 sizeof(RSA_DEC_IDS)/
 445                                                 sizeof(*RSA_DEC_IDS),
 446                                                 &alg_size, &func_id)) {
 447                                         return -(EINVAL);
 448                                 }
 449                                 alg_size_in_bytes = alg_size >> 3;
 450                                 rte_memcpy(cookie->input_array[1] +
 451                                                 alg_size_in_bytes -
 452                                                 xform->rsa.d.length,
 453                                                 xform->rsa.d.data,
 454                                                 xform->rsa.d.length);
 455                                 rte_memcpy(cookie->input_array[2] +
 456                                                 alg_size_in_bytes -
 457                                                 xform->rsa.n.length,
 458                                                 xform->rsa.n.data,
 459                                                 xform->rsa.n.length);
 460 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 461                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA ciphertext",
 462                                         cookie->input_array[0],
 463                                         alg_size_in_bytes);
 464                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA d", cookie->input_array[1],
 465                                         alg_size_in_bytes);
 466                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA n", cookie->input_array[2],
 467                                         alg_size_in_bytes);
 468 #endif
 469
 470                                 cookie->alg_size = alg_size;
 471                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 472                         } else {
 473                                 QAT_LOG(ERR, "Invalid RSA key type");
 474                                 return -(EINVAL);
 475                         }
 476                 }
 477         } else {
 478                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 479                 return -(EINVAL);
 480         }
 481         return 0;
 482 }
 483
 484 int
 485 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 486                 __rte_unused uint64_t *opaque,
 487                 __rte_unused enum qat_device_gen dev_gen)
 488 {
 489         struct qat_asym_session *ctx;
 490         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 491         struct rte_crypto_asym_op *asym_op = op->asym;
 492         struct icp_qat_fw_pke_request *qat_req =
 493                         (struct icp_qat_fw_pke_request *)out_msg;
 494         struct qat_asym_op_cookie *cookie =
 495                                 (struct qat_asym_op_cookie *)op_cookie;
 496         int err = 0;
 497
 498         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 499         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 500                 ctx = (struct qat_asym_session *)
 501                                 op->asym->session->sess_private_data;
 502                 if (unlikely(ctx == NULL)) {
 503                         QAT_LOG(ERR, "Session has not been created for this device");
 504                         goto error;
 505                 }
 506                 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
 507                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
 508                 if (err) {
 509                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 510                         goto error;
 511                 }
 512         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 513                 qat_fill_req_tmpl(qat_req);
 514                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
 515                                 op->asym->xform);
 516                 if (err) {
 517                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 518                         goto error;
 519                 }
 520         } else {
 521                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
 522                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 523                 goto error;
 524         }
 525
 526         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 527         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 528         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 529
 530 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 531         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 532                         sizeof(struct icp_qat_fw_pke_request));
 533 #endif
 534
 535         return 0;
 536 error:
 537
 538         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 539
 540 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 541         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 542                 sizeof(struct icp_qat_fw_pke_request));
 543 #endif
 544
 545         qat_req->output_param_count = 0;
 546         qat_req->input_param_count = 0;
 547         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 548         cookie->error |= err;
 549
 550         return 0;
 551 }
 552
 553 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
 554                 struct qat_asym_op_cookie *cookie,
 555                 struct rte_crypto_asym_xform *xform)
 556 {
 557         size_t alg_size, alg_size_in_bytes = 0;
 558         struct rte_crypto_asym_op *asym_op = rx_op->asym;
 559
 560         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 561                 rte_crypto_param n = xform->modex.modulus;
 562
 563                 alg_size = cookie->alg_size;
 564                 alg_size_in_bytes = alg_size >> 3;
 565                 uint8_t *modexp_result = asym_op->modex.result.data;
 566
 567                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 568                         rte_memcpy(modexp_result +
 569                                 (asym_op->modex.result.length -
 570                                         n.length),
 571                                 cookie->output_array[0] + alg_size_in_bytes
 572                                 - n.length, n.length
 573                                 );
 574                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 575 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 576                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
 577                                         cookie->output_array[0],
 578                                         alg_size_in_bytes);
 579
 580 #endif
 581                 }
 582         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 583                 rte_crypto_param n = xform->modinv.modulus;
 584
 585                 alg_size = cookie->alg_size;
 586                 alg_size_in_bytes = alg_size >> 3;
 587                 uint8_t *modinv_result = asym_op->modinv.result.data;
 588
 589                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 590                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 591                                 - n.length),
 592                                 cookie->output_array[0] + alg_size_in_bytes
 593                                 - n.length, n.length);
 594                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 595 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 596                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
 597                                         cookie->output_array[0],
 598                                         alg_size_in_bytes);
 599 #endif
 600                 }
 601         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 602
 603                 alg_size = cookie->alg_size;
 604                 alg_size_in_bytes = alg_size >> 3;
 605                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 606                                 asym_op->rsa.op_type ==
 607                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 608                         if (asym_op->rsa.op_type ==
 609                                         RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 610                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 611
 612                                 rte_memcpy(rsa_result,
 613                                                 cookie->output_array[0],
 614                                                 alg_size_in_bytes);
 615                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 616 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 617                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Encrypted data",
 618                                                 cookie->output_array[0],
 619                                                 alg_size_in_bytes);
 620 #endif
 621                         } else if (asym_op->rsa.op_type ==
 622                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 623                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 624
 625                                 switch (asym_op->rsa.pad) {
 626                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 627                                         rte_memcpy(rsa_result,
 628                                                         cookie->output_array[0],
 629                                                         alg_size_in_bytes);
 630                                         rx_op->status =
 631                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 632 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 633                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 634                                                 cookie->output_array[0],
 635                                                 alg_size_in_bytes);
 636 #endif
 637                                         break;
 638                                 default:
 639                                         QAT_LOG(ERR, "Padding not supported");
 640                                         rx_op->status =
 641                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 642                                         break;
 643                                 }
 644                         }
 645                 } else {
 646                         if (asym_op->rsa.op_type ==
 647                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 648                                 uint8_t *rsa_result = asym_op->rsa.message.data;
 649
 650                                 switch (asym_op->rsa.pad) {
 651                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 652                                         rte_memcpy(rsa_result,
 653                                                 cookie->output_array[0],
 654                                                 alg_size_in_bytes);
 655                                         rx_op->status =
 656                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 657                                         break;
 658                                 default:
 659                                         QAT_LOG(ERR, "Padding not supported");
 660                                         rx_op->status =
 661                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 662                                         break;
 663                                 }
 664 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 665                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Decrypted Message",
 666                                                 rsa_result, alg_size_in_bytes);
 667 #endif
 668                         } else if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
 669                                 uint8_t *rsa_result = asym_op->rsa.sign.data;
 670
 671                                 rte_memcpy(rsa_result,
 672                                                 cookie->output_array[0],
 673                                                 alg_size_in_bytes);
 674                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 675 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 676                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 677                                                 cookie->output_array[0],
 678                                                 alg_size_in_bytes);
 679 #endif
 680                         }
 681                 }
 682         }
 683         qat_clear_arrays_by_alg(cookie, xform, alg_size_in_bytes);
 684 }
 685
 686 int
 687 qat_asym_process_response(void **op, uint8_t *resp,
 688                 void *op_cookie, __rte_unused uint64_t *dequeue_err_count)
 689 {
 690         struct qat_asym_session *ctx;
 691         struct icp_qat_fw_pke_resp *resp_msg =
 692                         (struct icp_qat_fw_pke_resp *)resp;
 693         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
 694                         (resp_msg->opaque);
 695         struct qat_asym_op_cookie *cookie = op_cookie;
 696
 697         if (cookie->error) {
 698                 cookie->error = 0;
 699                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 700                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 701                 QAT_DP_LOG(ERR, "Cookie status returned error");
 702         } else {
 703                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 704                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 705                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 706                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 707                         QAT_DP_LOG(ERR, "Asymmetric response status"
 708                                         " returned error");
 709                 }
 710                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 711                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 712                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 713                         QAT_DP_LOG(ERR, "Asymmetric common status"
 714                                         " returned error");
 715                 }
 716         }
 717
 718         if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 719                 ctx = (struct qat_asym_session *)
 720                                 rx_op->asym->session->sess_private_data;
 721                 qat_asym_collect_response(rx_op, cookie, ctx->xform);
 722         } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 723                 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
 724         }
 725         *op = rx_op;
 726
 727 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 728         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
 729                         sizeof(struct icp_qat_fw_pke_resp));
 730 #endif
 731
 732         return 1;
 733 }
 734
 735 int
 736 qat_asym_session_configure(struct rte_cryptodev *dev __rte_unused,
 737                 struct rte_crypto_asym_xform *xform,
 738                 struct rte_cryptodev_asym_session *sess)
 739 {
 740         struct qat_asym_session *session;
 741
 742         session = (struct qat_asym_session *) sess->sess_private_data;
 743         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 744                 if (xform->modex.exponent.length == 0 ||
 745                                 xform->modex.modulus.length == 0) {
 746                         QAT_LOG(ERR, "Invalid mod exp input parameter");
 747                         return -EINVAL;
 748                 }
 749         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 750                 if (xform->modinv.modulus.length == 0) {
 751                         QAT_LOG(ERR, "Invalid mod inv input parameter");
 752                         return -EINVAL;
 753                 }
 754         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 755                 if (xform->rsa.n.length == 0) {
 756                         QAT_LOG(ERR, "Invalid rsa input parameter");
 757                         return -EINVAL;
 758                 }
 759         } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
 760                         || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
 761                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 762                 return -EINVAL;
 763         } else {
 764                 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
 765                 return -EINVAL;
 766         }
 767
 768         session->xform = xform;
 769         qat_asym_build_req_tmpl(session);
 770
 771         return 0;
 772 }
 773
 774 unsigned int qat_asym_session_get_private_size(
 775                 struct rte_cryptodev *dev __rte_unused)
 776 {
 777         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
 778 }
 779
 780 void
 781 qat_asym_session_clear(struct rte_cryptodev *dev,
 782                 struct rte_cryptodev_asym_session *sess)
 783 {
 784         void *sess_priv = sess->sess_private_data;
 785         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
 786
 787         if (sess_priv)
 788                 memset(s, 0, qat_asym_session_get_private_size(dev));
 789 }
 790
 791 static struct cryptodev_driver qat_crypto_drv;
 792 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 793                 cryptodev_qat_asym_driver,
 794                 qat_asym_driver_id);