drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 - 2022 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include <cryptodev_pmd.h>
   8
   9 #include "icp_qat_fw_pke.h"
  10 #include "icp_qat_fw.h"
  11 #include "qat_pke_functionality_arrays.h"
  12
  13 #include "qat_device.h"
  14
  15 #include "qat_logs.h"
  16 #include "qat_asym.h"
  17
  18 uint8_t qat_asym_driver_id;
  19
  20 struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[QAT_N_GENS];
  21
  22 void
  23 qat_asym_init_op_cookie(void *op_cookie)
  24 {
  25         int j;
  26         struct qat_asym_op_cookie *cookie = op_cookie;
  27
  28         cookie->input_addr = rte_mempool_virt2iova(cookie) +
  29                         offsetof(struct qat_asym_op_cookie,
  30                                         input_params_ptrs);
  31
  32         cookie->output_addr = rte_mempool_virt2iova(cookie) +
  33                         offsetof(struct qat_asym_op_cookie,
  34                                         output_params_ptrs);
  35
  36         for (j = 0; j < 8; j++) {
  37                 cookie->input_params_ptrs[j] =
  38                                 rte_mempool_virt2iova(cookie) +
  39                                 offsetof(struct qat_asym_op_cookie,
  40                                                 input_array[j]);
  41                 cookie->output_params_ptrs[j] =
  42                                 rte_mempool_virt2iova(cookie) +
  43                                 offsetof(struct qat_asym_op_cookie,
  44                                                 output_array[j]);
  45         }
  46 }
  47
  48 /* An rte_driver is needed in the registration of both the device and the driver
  49  * with cryptodev.
  50  * The actual qat pci's rte_driver can't be used as its name represents
  51  * the whole pci device with all services. Think of this as a holder for a name
  52  * for the crypto part of the pci device.
  53  */
  54 static const char qat_asym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_ASYM_PMD);
  55 static const struct rte_driver cryptodev_qat_asym_driver = {
  56         .name = qat_asym_drv_name,
  57         .alias = qat_asym_drv_name
  58 };
  59
  60
  61 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
  62                 int in_count, int out_count, int alg_size)
  63 {
  64         int i;
  65
  66         for (i = 0; i < in_count; i++)
  67                 memset(cookie->input_array[i], 0x0, alg_size);
  68         for (i = 0; i < out_count; i++)
  69                 memset(cookie->output_array[i], 0x0, alg_size);
  70 }
  71
  72 static void qat_clear_arrays_crt(struct qat_asym_op_cookie *cookie,
  73                 int alg_size)
  74 {
  75         int i;
  76
  77         memset(cookie->input_array[0], 0x0, alg_size);
  78         for (i = 1; i < QAT_ASYM_RSA_QT_NUM_IN_PARAMS; i++)
  79                 memset(cookie->input_array[i], 0x0, alg_size / 2);
  80         for (i = 0; i < QAT_ASYM_RSA_NUM_OUT_PARAMS; i++)
  81                 memset(cookie->output_array[i], 0x0, alg_size);
  82 }
  83
  84 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
  85                 struct rte_crypto_asym_xform *xform, int alg_size)
  86 {
  87         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX)
  88                 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
  89                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, alg_size);
  90         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV)
  91                 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
  92                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, alg_size);
  93         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
  94                 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT)
  95                         qat_clear_arrays_crt(cookie, alg_size);
  96                 else {
  97                         qat_clear_arrays(cookie, QAT_ASYM_RSA_NUM_IN_PARAMS,
  98                                 QAT_ASYM_RSA_NUM_OUT_PARAMS, alg_size);
  99                 }
 100         }
 101 }
 102
 103 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
 104
 105 static int
 106 qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
 107                 size_t arr_sz, size_t *size, uint32_t *func_id)
 108 {
 109         size_t i;
 110
 111         for (i = 0; i < arr_sz; i++) {
 112                 if (*size <= arr[i][0]) {
 113                         *size = arr[i][0];
 114                         *func_id = arr[i][1];
 115                         return 0;
 116                 }
 117         }
 118         return -1;
 119 }
 120
 121 static size_t
 122 max_of(int n, ...)
 123 {
 124         va_list args;
 125         size_t len = 0, num;
 126         int i;
 127
 128         va_start(args, n);
 129         len = va_arg(args, size_t);
 130
 131         for (i = 0; i < n - 1; i++) {
 132                 num = va_arg(args, size_t);
 133                 if (num > len)
 134                         len = num;
 135         }
 136         va_end(args);
 137
 138         return len;
 139 }
 140
 141 static int
 142 qat_asym_check_nonzero(rte_crypto_param n)
 143 {
 144         if (n.length < 8) {
 145                 /* Not a case for any cryptographic function except for DH
 146                  * generator which very often can be of one byte length
 147                  */
 148                 size_t i;
 149
 150                 if (n.data[n.length - 1] == 0x0) {
 151                         for (i = 0; i < n.length - 1; i++)
 152                                 if (n.data[i] != 0x0)
 153                                         break;
 154                         if (i == n.length - 1)
 155                                 return -(EINVAL);
 156                 }
 157         } else if (*(uint64_t *)&n.data[
 158                                 n.length - 8] == 0) {
 159                 /* Very likely it is zeroed modulus */
 160                 size_t i;
 161
 162                 for (i = 0; i < n.length - 8; i++)
 163                         if (n.data[i] != 0x0)
 164                                 break;
 165                 if (i == n.length - 8)
 166                         return -(EINVAL);
 167         }
 168
 169         return 0;
 170 }
 171
 172 static int
 173 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 174                 struct icp_qat_fw_pke_request *qat_req,
 175                 struct qat_asym_op_cookie *cookie,
 176                 struct rte_crypto_asym_xform *xform)
 177 {
 178         int err = 0;
 179         size_t alg_size;
 180         size_t alg_size_in_bytes;
 181         uint32_t func_id = 0;
 182
 183         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 184                 err = qat_asym_check_nonzero(xform->modex.modulus);
 185                 if (err) {
 186                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
 187                                         " aborting this operation");
 188                         return err;
 189                 }
 190
 191                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
 192                                xform->modex.exponent.length,
 193                                xform->modex.modulus.length);
 194                 alg_size = alg_size_in_bytes << 3;
 195
 196                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
 197                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
 198                                 &alg_size, &func_id)) {
 199                         return -(EINVAL);
 200                 }
 201
 202                 alg_size_in_bytes = alg_size >> 3;
 203                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 204                         asym_op->modex.base.length
 205                         , asym_op->modex.base.data,
 206                         asym_op->modex.base.length);
 207                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 208                         xform->modex.exponent.length
 209                         , xform->modex.exponent.data,
 210                         xform->modex.exponent.length);
 211                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
 212                         xform->modex.modulus.length,
 213                         xform->modex.modulus.data,
 214                         xform->modex.modulus.length);
 215                 cookie->alg_size = alg_size;
 216                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 217                 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
 218                 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
 219 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 220                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
 221                                 cookie->input_array[0],
 222                                 alg_size_in_bytes);
 223                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
 224                                 cookie->input_array[1],
 225                                 alg_size_in_bytes);
 226                 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
 227                                 cookie->input_array[2],
 228                                 alg_size_in_bytes);
 229 #endif
 230         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 231                 err = qat_asym_check_nonzero(xform->modinv.modulus);
 232                 if (err) {
 233                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
 234                                         " inverse, aborting this operation");
 235                         return err;
 236                 }
 237
 238                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
 239                                 xform->modinv.modulus.length);
 240                 alg_size = alg_size_in_bytes << 3;
 241
 242                 if (xform->modinv.modulus.data[
 243                                 xform->modinv.modulus.length - 1] & 0x01) {
 244                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
 245                                         sizeof(MOD_INV_IDS_ODD)/
 246                                         sizeof(*MOD_INV_IDS_ODD),
 247                                         &alg_size, &func_id)) {
 248                                 return -(EINVAL);
 249                         }
 250                 } else {
 251                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
 252                                         sizeof(MOD_INV_IDS_EVEN)/
 253                                         sizeof(*MOD_INV_IDS_EVEN),
 254                                         &alg_size, &func_id)) {
 255                                 return -(EINVAL);
 256                         }
 257                 }
 258
 259                 alg_size_in_bytes = alg_size >> 3;
 260                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 261                         asym_op->modinv.base.length
 262                                 , asym_op->modinv.base.data,
 263                                 asym_op->modinv.base.length);
 264                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 265                                 xform->modinv.modulus.length
 266                                 , xform->modinv.modulus.data,
 267                                 xform->modinv.modulus.length);
 268                 cookie->alg_size = alg_size;
 269                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 270                 qat_req->input_param_count =
 271                                 QAT_ASYM_MODINV_NUM_IN_PARAMS;
 272                 qat_req->output_param_count =
 273                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
 274 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 275                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
 276                                 cookie->input_array[0],
 277                                 alg_size_in_bytes);
 278                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
 279                                 cookie->input_array[1],
 280                                 alg_size_in_bytes);
 281 #endif
 282         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 283                 err = qat_asym_check_nonzero(xform->rsa.n);
 284                 if (err) {
 285                         QAT_LOG(ERR, "Empty modulus in RSA"
 286                                         " inverse, aborting this operation");
 287                         return err;
 288                 }
 289
 290                 alg_size_in_bytes = xform->rsa.n.length;
 291                 alg_size = alg_size_in_bytes << 3;
 292
 293                 qat_req->input_param_count =
 294                                 QAT_ASYM_RSA_NUM_IN_PARAMS;
 295                 qat_req->output_param_count =
 296                                 QAT_ASYM_RSA_NUM_OUT_PARAMS;
 297
 298                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 299                                 asym_op->rsa.op_type ==
 300                                                 RTE_CRYPTO_ASYM_OP_VERIFY) {
 301
 302                         if (qat_asym_get_sz_and_func_id(RSA_ENC_IDS,
 303                                         sizeof(RSA_ENC_IDS)/
 304                                         sizeof(*RSA_ENC_IDS),
 305                                         &alg_size, &func_id)) {
 306                                 err = -(EINVAL);
 307                                 QAT_LOG(ERR,
 308                                         "Not supported RSA parameter size (key)");
 309                                 return err;
 310                         }
 311                         alg_size_in_bytes = alg_size >> 3;
 312                         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 313                                 switch (asym_op->rsa.pad) {
 314                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 315                                         rte_memcpy(cookie->input_array[0] +
 316                                                 alg_size_in_bytes -
 317                                                 asym_op->rsa.message.length
 318                                                 , asym_op->rsa.message.data,
 319                                                 asym_op->rsa.message.length);
 320                                         break;
 321                                 default:
 322                                         err = -(EINVAL);
 323                                         QAT_LOG(ERR,
 324                                                 "Invalid RSA padding (Encryption)");
 325                                         return err;
 326                                 }
 327 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 328                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Message",
 329                                                 cookie->input_array[0],
 330                                                 alg_size_in_bytes);
 331 #endif
 332                         } else {
 333                                 switch (asym_op->rsa.pad) {
 334                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 335                                         rte_memcpy(cookie->input_array[0],
 336                                                 asym_op->rsa.sign.data,
 337                                                 alg_size_in_bytes);
 338                                         break;
 339                                 default:
 340                                         err = -(EINVAL);
 341                                         QAT_LOG(ERR,
 342                                                 "Invalid RSA padding (Verify)");
 343                                         return err;
 344                                 }
 345
 346 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 347                                 QAT_DP_HEXDUMP_LOG(DEBUG, " RSA Signature",
 348                                                 cookie->input_array[0],
 349                                                 alg_size_in_bytes);
 350 #endif
 351
 352                         }
 353                         rte_memcpy(cookie->input_array[1] +
 354                                         alg_size_in_bytes -
 355                                         xform->rsa.e.length
 356                                         , xform->rsa.e.data,
 357                                         xform->rsa.e.length);
 358                         rte_memcpy(cookie->input_array[2] +
 359                                         alg_size_in_bytes -
 360                                         xform->rsa.n.length,
 361                                         xform->rsa.n.data,
 362                                         xform->rsa.n.length);
 363
 364                         cookie->alg_size = alg_size;
 365                         qat_req->pke_hdr.cd_pars.func_id = func_id;
 366
 367 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 368                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Public Key",
 369                                         cookie->input_array[1], alg_size_in_bytes);
 370                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Modulus",
 371                                         cookie->input_array[2], alg_size_in_bytes);
 372 #endif
 373                 } else {
 374                         if (asym_op->rsa.op_type ==
 375                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 376                                 switch (asym_op->rsa.pad) {
 377                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 378                                         rte_memcpy(cookie->input_array[0]
 379                                                 + alg_size_in_bytes -
 380                                                 asym_op->rsa.cipher.length,
 381                                                 asym_op->rsa.cipher.data,
 382                                                 asym_op->rsa.cipher.length);
 383                                         break;
 384                                 default:
 385                                         QAT_LOG(ERR,
 386                                                 "Invalid padding of RSA (Decrypt)");
 387                                         return -(EINVAL);
 388                                 }
 389
 390                         } else if (asym_op->rsa.op_type ==
 391                                         RTE_CRYPTO_ASYM_OP_SIGN) {
 392                                 switch (asym_op->rsa.pad) {
 393                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 394                                         rte_memcpy(cookie->input_array[0]
 395                                                 + alg_size_in_bytes -
 396                                                 asym_op->rsa.message.length,
 397                                                 asym_op->rsa.message.data,
 398                                                 asym_op->rsa.message.length);
 399                                         break;
 400                                 default:
 401                                         QAT_LOG(ERR,
 402                                                 "Invalid padding of RSA (Signature)");
 403                                         return -(EINVAL);
 404                                 }
 405                         }
 406                         if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT) {
 407
 408                                 qat_req->input_param_count =
 409                                                 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
 410                                 if (qat_asym_get_sz_and_func_id(RSA_DEC_CRT_IDS,
 411                                                 sizeof(RSA_DEC_CRT_IDS)/
 412                                                 sizeof(*RSA_DEC_CRT_IDS),
 413                                                 &alg_size, &func_id)) {
 414                                         return -(EINVAL);
 415                                 }
 416                                 alg_size_in_bytes = alg_size >> 3;
 417
 418                                 rte_memcpy(cookie->input_array[1] +
 419                                                 (alg_size_in_bytes >> 1) -
 420                                                 xform->rsa.qt.p.length
 421                                                 , xform->rsa.qt.p.data,
 422                                                 xform->rsa.qt.p.length);
 423                                 rte_memcpy(cookie->input_array[2] +
 424                                                 (alg_size_in_bytes >> 1) -
 425                                                 xform->rsa.qt.q.length
 426                                                 , xform->rsa.qt.q.data,
 427                                                 xform->rsa.qt.q.length);
 428                                 rte_memcpy(cookie->input_array[3] +
 429                                                 (alg_size_in_bytes >> 1) -
 430                                                 xform->rsa.qt.dP.length
 431                                                 , xform->rsa.qt.dP.data,
 432                                                 xform->rsa.qt.dP.length);
 433                                 rte_memcpy(cookie->input_array[4] +
 434                                                 (alg_size_in_bytes >> 1) -
 435                                                 xform->rsa.qt.dQ.length
 436                                                 , xform->rsa.qt.dQ.data,
 437                                                 xform->rsa.qt.dQ.length);
 438                                 rte_memcpy(cookie->input_array[5] +
 439                                                 (alg_size_in_bytes >> 1) -
 440                                                 xform->rsa.qt.qInv.length
 441                                                 , xform->rsa.qt.qInv.data,
 442                                                 xform->rsa.qt.qInv.length);
 443                                 cookie->alg_size = alg_size;
 444                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 445
 446 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 447                                 QAT_DP_HEXDUMP_LOG(DEBUG, "C",
 448                                                 cookie->input_array[0],
 449                                                 alg_size_in_bytes);
 450                                 QAT_DP_HEXDUMP_LOG(DEBUG, "p",
 451                                                 cookie->input_array[1],
 452                                                 alg_size_in_bytes);
 453                                 QAT_DP_HEXDUMP_LOG(DEBUG, "q",
 454                                                 cookie->input_array[2],
 455                                                 alg_size_in_bytes);
 456                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 457                                                 "dP", cookie->input_array[3],
 458                                                 alg_size_in_bytes);
 459                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 460                                                 "dQ", cookie->input_array[4],
 461                                                 alg_size_in_bytes);
 462                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 463                                                 "qInv", cookie->input_array[5],
 464                                                 alg_size_in_bytes);
 465 #endif
 466                         } else if (xform->rsa.key_type ==
 467                                         RTE_RSA_KEY_TYPE_EXP) {
 468                                 if (qat_asym_get_sz_and_func_id(
 469                                                 RSA_DEC_IDS,
 470                                                 sizeof(RSA_DEC_IDS)/
 471                                                 sizeof(*RSA_DEC_IDS),
 472                                                 &alg_size, &func_id)) {
 473                                         return -(EINVAL);
 474                                 }
 475                                 alg_size_in_bytes = alg_size >> 3;
 476                                 rte_memcpy(cookie->input_array[1] +
 477                                                 alg_size_in_bytes -
 478                                                 xform->rsa.d.length,
 479                                                 xform->rsa.d.data,
 480                                                 xform->rsa.d.length);
 481                                 rte_memcpy(cookie->input_array[2] +
 482                                                 alg_size_in_bytes -
 483                                                 xform->rsa.n.length,
 484                                                 xform->rsa.n.data,
 485                                                 xform->rsa.n.length);
 486 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 487                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA ciphertext",
 488                                         cookie->input_array[0],
 489                                         alg_size_in_bytes);
 490                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA d", cookie->input_array[1],
 491                                         alg_size_in_bytes);
 492                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA n", cookie->input_array[2],
 493                                         alg_size_in_bytes);
 494 #endif
 495
 496                                 cookie->alg_size = alg_size;
 497                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 498                         } else {
 499                                 QAT_LOG(ERR, "Invalid RSA key type");
 500                                 return -(EINVAL);
 501                         }
 502                 }
 503         } else {
 504                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 505                 return -(EINVAL);
 506         }
 507         return 0;
 508 }
 509
 510 int
 511 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 512                 __rte_unused uint64_t *opaque,
 513                 __rte_unused enum qat_device_gen dev_gen)
 514 {
 515         struct qat_asym_session *ctx;
 516         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 517         struct rte_crypto_asym_op *asym_op = op->asym;
 518         struct icp_qat_fw_pke_request *qat_req =
 519                         (struct icp_qat_fw_pke_request *)out_msg;
 520         struct qat_asym_op_cookie *cookie =
 521                                 (struct qat_asym_op_cookie *)op_cookie;
 522         int err = 0;
 523
 524         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 525         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 526                 ctx = (struct qat_asym_session *)
 527                                 op->asym->session->sess_private_data;
 528                 if (unlikely(ctx == NULL)) {
 529                         QAT_LOG(ERR, "Session has not been created for this device");
 530                         goto error;
 531                 }
 532                 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
 533                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
 534                 if (err) {
 535                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 536                         goto error;
 537                 }
 538         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 539                 qat_fill_req_tmpl(qat_req);
 540                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
 541                                 op->asym->xform);
 542                 if (err) {
 543                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 544                         goto error;
 545                 }
 546         } else {
 547                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
 548                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 549                 goto error;
 550         }
 551
 552         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 553         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 554         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 555
 556 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 557         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 558                         sizeof(struct icp_qat_fw_pke_request));
 559 #endif
 560
 561         return 0;
 562 error:
 563
 564         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 565
 566 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 567         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 568                 sizeof(struct icp_qat_fw_pke_request));
 569 #endif
 570
 571         qat_req->output_param_count = 0;
 572         qat_req->input_param_count = 0;
 573         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 574         cookie->error |= err;
 575
 576         return 0;
 577 }
 578
 579 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
 580                 struct qat_asym_op_cookie *cookie,
 581                 struct rte_crypto_asym_xform *xform)
 582 {
 583         size_t alg_size, alg_size_in_bytes = 0;
 584         struct rte_crypto_asym_op *asym_op = rx_op->asym;
 585
 586         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 587                 rte_crypto_param n = xform->modex.modulus;
 588
 589                 alg_size = cookie->alg_size;
 590                 alg_size_in_bytes = alg_size >> 3;
 591                 uint8_t *modexp_result = asym_op->modex.result.data;
 592
 593                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 594                         rte_memcpy(modexp_result +
 595                                 (asym_op->modex.result.length -
 596                                         n.length),
 597                                 cookie->output_array[0] + alg_size_in_bytes
 598                                 - n.length, n.length
 599                                 );
 600                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 601 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 602                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
 603                                         cookie->output_array[0],
 604                                         alg_size_in_bytes);
 605
 606 #endif
 607                 }
 608         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 609                 rte_crypto_param n = xform->modinv.modulus;
 610
 611                 alg_size = cookie->alg_size;
 612                 alg_size_in_bytes = alg_size >> 3;
 613                 uint8_t *modinv_result = asym_op->modinv.result.data;
 614
 615                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 616                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 617                                 - n.length),
 618                                 cookie->output_array[0] + alg_size_in_bytes
 619                                 - n.length, n.length);
 620                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 621 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 622                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
 623                                         cookie->output_array[0],
 624                                         alg_size_in_bytes);
 625 #endif
 626                 }
 627         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 628
 629                 alg_size = cookie->alg_size;
 630                 alg_size_in_bytes = alg_size >> 3;
 631                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 632                                 asym_op->rsa.op_type ==
 633                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 634                         if (asym_op->rsa.op_type ==
 635                                         RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 636                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 637
 638                                 rte_memcpy(rsa_result,
 639                                                 cookie->output_array[0],
 640                                                 alg_size_in_bytes);
 641                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 642 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 643                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Encrypted data",
 644                                                 cookie->output_array[0],
 645                                                 alg_size_in_bytes);
 646 #endif
 647                         } else if (asym_op->rsa.op_type ==
 648                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 649                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 650
 651                                 switch (asym_op->rsa.pad) {
 652                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 653                                         rte_memcpy(rsa_result,
 654                                                         cookie->output_array[0],
 655                                                         alg_size_in_bytes);
 656                                         rx_op->status =
 657                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 658 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 659                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 660                                                 cookie->output_array[0],
 661                                                 alg_size_in_bytes);
 662 #endif
 663                                         break;
 664                                 default:
 665                                         QAT_LOG(ERR, "Padding not supported");
 666                                         rx_op->status =
 667                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 668                                         break;
 669                                 }
 670                         }
 671                 } else {
 672                         if (asym_op->rsa.op_type ==
 673                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 674                                 uint8_t *rsa_result = asym_op->rsa.message.data;
 675
 676                                 switch (asym_op->rsa.pad) {
 677                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 678                                         rte_memcpy(rsa_result,
 679                                                 cookie->output_array[0],
 680                                                 alg_size_in_bytes);
 681                                         rx_op->status =
 682                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 683                                         break;
 684                                 default:
 685                                         QAT_LOG(ERR, "Padding not supported");
 686                                         rx_op->status =
 687                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 688                                         break;
 689                                 }
 690 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 691                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Decrypted Message",
 692                                                 rsa_result, alg_size_in_bytes);
 693 #endif
 694                         } else if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
 695                                 uint8_t *rsa_result = asym_op->rsa.sign.data;
 696
 697                                 rte_memcpy(rsa_result,
 698                                                 cookie->output_array[0],
 699                                                 alg_size_in_bytes);
 700                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 701 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 702                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 703                                                 cookie->output_array[0],
 704                                                 alg_size_in_bytes);
 705 #endif
 706                         }
 707                 }
 708         }
 709         qat_clear_arrays_by_alg(cookie, xform, alg_size_in_bytes);
 710 }
 711
 712 int
 713 qat_asym_process_response(void **op, uint8_t *resp,
 714                 void *op_cookie, __rte_unused uint64_t *dequeue_err_count)
 715 {
 716         struct qat_asym_session *ctx;
 717         struct icp_qat_fw_pke_resp *resp_msg =
 718                         (struct icp_qat_fw_pke_resp *)resp;
 719         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
 720                         (resp_msg->opaque);
 721         struct qat_asym_op_cookie *cookie = op_cookie;
 722
 723         if (cookie->error) {
 724                 cookie->error = 0;
 725                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 726                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 727                 QAT_DP_LOG(ERR, "Cookie status returned error");
 728         } else {
 729                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 730                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 731                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 732                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 733                         QAT_DP_LOG(ERR, "Asymmetric response status"
 734                                         " returned error");
 735                 }
 736                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 737                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 738                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 739                         QAT_DP_LOG(ERR, "Asymmetric common status"
 740                                         " returned error");
 741                 }
 742         }
 743
 744         if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 745                 ctx = (struct qat_asym_session *)
 746                                 rx_op->asym->session->sess_private_data;
 747                 qat_asym_collect_response(rx_op, cookie, ctx->xform);
 748         } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 749                 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
 750         }
 751         *op = rx_op;
 752
 753 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 754         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
 755                         sizeof(struct icp_qat_fw_pke_resp));
 756 #endif
 757
 758         return 1;
 759 }
 760
 761 int
 762 qat_asym_session_configure(struct rte_cryptodev *dev __rte_unused,
 763                 struct rte_crypto_asym_xform *xform,
 764                 struct rte_cryptodev_asym_session *sess)
 765 {
 766         struct qat_asym_session *session;
 767
 768         session = (struct qat_asym_session *) sess->sess_private_data;
 769         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 770                 if (xform->modex.exponent.length == 0 ||
 771                                 xform->modex.modulus.length == 0) {
 772                         QAT_LOG(ERR, "Invalid mod exp input parameter");
 773                         return -EINVAL;
 774                 }
 775         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 776                 if (xform->modinv.modulus.length == 0) {
 777                         QAT_LOG(ERR, "Invalid mod inv input parameter");
 778                         return -EINVAL;
 779                 }
 780         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 781                 if (xform->rsa.n.length == 0) {
 782                         QAT_LOG(ERR, "Invalid rsa input parameter");
 783                         return -EINVAL;
 784                 }
 785         } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
 786                         || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
 787                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 788                 return -EINVAL;
 789         } else {
 790                 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
 791                 return -EINVAL;
 792         }
 793
 794         session->xform = xform;
 795         qat_asym_build_req_tmpl(session);
 796
 797         return 0;
 798 }
 799
 800 unsigned int qat_asym_session_get_private_size(
 801                 struct rte_cryptodev *dev __rte_unused)
 802 {
 803         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
 804 }
 805
 806 void
 807 qat_asym_session_clear(struct rte_cryptodev *dev,
 808                 struct rte_cryptodev_asym_session *sess)
 809 {
 810         void *sess_priv = sess->sess_private_data;
 811         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
 812
 813         if (sess_priv)
 814                 memset(s, 0, qat_asym_session_get_private_size(dev));
 815 }
 816
 817 static uint16_t
 818 qat_asym_crypto_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 819                 uint16_t nb_ops)
 820 {
 821         return qat_enqueue_op_burst(qp, qat_asym_build_request, (void **)ops,
 822                         nb_ops);
 823 }
 824
 825 static uint16_t
 826 qat_asym_crypto_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 827                 uint16_t nb_ops)
 828 {
 829         return qat_dequeue_op_burst(qp, (void **)ops, qat_asym_process_response,
 830                                 nb_ops);
 831 }
 832
 833 int
 834 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
 835                 struct qat_dev_cmd_param *qat_dev_cmd_param)
 836 {
 837         struct qat_cryptodev_private *internals;
 838         struct rte_cryptodev *cryptodev;
 839         struct qat_device_info *qat_dev_instance =
 840                 &qat_pci_devs[qat_pci_dev->qat_dev_id];
 841         struct rte_cryptodev_pmd_init_params init_params = {
 842                 .name = "",
 843                 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
 844                 .private_data_size = sizeof(struct qat_cryptodev_private)
 845         };
 846         struct qat_capabilities_info capa_info;
 847         const struct rte_cryptodev_capabilities *capabilities;
 848         const struct qat_crypto_gen_dev_ops *gen_dev_ops =
 849                 &qat_asym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
 850         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 851         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 852         uint64_t capa_size;
 853         int i = 0;
 854
 855         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
 856                         qat_pci_dev->name, "asym");
 857         QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name);
 858
 859         if (gen_dev_ops->cryptodev_ops == NULL) {
 860                 QAT_LOG(ERR, "Device %s does not support asymmetric crypto",
 861                                 name);
 862                 return -(EFAULT);
 863         }
 864
 865         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 866                 qat_pci_dev->qat_asym_driver_id =
 867                                 qat_asym_driver_id;
 868         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 869                 if (qat_pci_dev->qat_asym_driver_id !=
 870                                 qat_asym_driver_id) {
 871                         QAT_LOG(ERR,
 872                                 "Device %s have different driver id than corresponding device in primary process",
 873                                 name);
 874                         return -(EFAULT);
 875                 }
 876         }
 877
 878         /* Populate subset device to use in cryptodev device creation */
 879         qat_dev_instance->asym_rte_dev.driver = &cryptodev_qat_asym_driver;
 880         qat_dev_instance->asym_rte_dev.numa_node =
 881                         qat_dev_instance->pci_dev->device.numa_node;
 882         qat_dev_instance->asym_rte_dev.devargs = NULL;
 883
 884         cryptodev = rte_cryptodev_pmd_create(name,
 885                         &(qat_dev_instance->asym_rte_dev), &init_params);
 886
 887         if (cryptodev == NULL)
 888                 return -ENODEV;
 889
 890         qat_dev_instance->asym_rte_dev.name = cryptodev->data->name;
 891         cryptodev->driver_id = qat_asym_driver_id;
 892         cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
 893
 894         cryptodev->enqueue_burst = qat_asym_crypto_enqueue_op_burst;
 895         cryptodev->dequeue_burst = qat_asym_crypto_dequeue_op_burst;
 896
 897         cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
 898
 899         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 900                 return 0;
 901
 902         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 903                         "QAT_ASYM_CAPA_GEN_%d",
 904                         qat_pci_dev->qat_dev_gen);
 905
 906         internals = cryptodev->data->dev_private;
 907         internals->qat_dev = qat_pci_dev;
 908         internals->dev_id = cryptodev->data->dev_id;
 909
 910         capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
 911         capabilities = capa_info.data;
 912         capa_size = capa_info.size;
 913
 914         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
 915         if (internals->capa_mz == NULL) {
 916                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
 917                                 capa_size, rte_socket_id(), 0);
 918                 if (internals->capa_mz == NULL) {
 919                         QAT_LOG(DEBUG,
 920                                 "Error allocating memzone for capabilities, "
 921                                 "destroying PMD for %s",
 922                                 name);
 923                         rte_cryptodev_pmd_destroy(cryptodev);
 924                         memset(&qat_dev_instance->asym_rte_dev, 0,
 925                                 sizeof(qat_dev_instance->asym_rte_dev));
 926                         return -EFAULT;
 927                 }
 928         }
 929
 930         memcpy(internals->capa_mz->addr, capabilities, capa_size);
 931         internals->qat_dev_capabilities = internals->capa_mz->addr;
 932
 933         while (1) {
 934                 if (qat_dev_cmd_param[i].name == NULL)
 935                         break;
 936                 if (!strcmp(qat_dev_cmd_param[i].name, ASYM_ENQ_THRESHOLD_NAME))
 937                         internals->min_enq_burst_threshold =
 938                                         qat_dev_cmd_param[i].val;
 939                 i++;
 940         }
 941
 942         qat_pci_dev->asym_dev = internals;
 943         internals->service_type = QAT_SERVICE_ASYMMETRIC;
 944         QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
 945                         cryptodev->data->name, internals->dev_id);
 946         return 0;
 947 }
 948
 949 int
 950 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
 951 {
 952         struct rte_cryptodev *cryptodev;
 953
 954         if (qat_pci_dev == NULL)
 955                 return -ENODEV;
 956         if (qat_pci_dev->asym_dev == NULL)
 957                 return 0;
 958         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 959                 rte_memzone_free(qat_pci_dev->asym_dev->capa_mz);
 960
 961         /* free crypto device */
 962         cryptodev = rte_cryptodev_pmd_get_dev(
 963                         qat_pci_dev->asym_dev->dev_id);
 964         rte_cryptodev_pmd_destroy(cryptodev);
 965         qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
 966         qat_pci_dev->asym_dev = NULL;
 967
 968         return 0;
 969 }
 970
 971 static struct cryptodev_driver qat_crypto_drv;
 972 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 973                 cryptodev_qat_asym_driver,
 974                 qat_asym_driver_id);