drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 - 2022 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include <cryptodev_pmd.h>
   8
   9 #include "qat_device.h"
  10 #include "qat_logs.h"
  11
  12 #include "qat_asym.h"
  13 #include "icp_qat_fw_pke.h"
  14 #include "icp_qat_fw.h"
  15 #include "qat_pke.h"
  16 #include "qat_ec.h"
  17
  18 uint8_t qat_asym_driver_id;
  19
  20 struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[QAT_N_GENS];
  21
  22 /* An rte_driver is needed in the registration of both the device and the driver
  23  * with cryptodev.
  24  * The actual qat pci's rte_driver can't be used as its name represents
  25  * the whole pci device with all services. Think of this as a holder for a name
  26  * for the crypto part of the pci device.
  27  */
  28 static const char qat_asym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_ASYM_PMD);
  29 static const struct rte_driver cryptodev_qat_asym_driver = {
  30         .name = qat_asym_drv_name,
  31         .alias = qat_asym_drv_name
  32 };
  33
  34 /*
  35  * Macros with suffix _F are used with some of predefinded identifiers:
  36  * - cookie->input_buffer
  37  * - qat_func_alignsize
  38  */
  39 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
  40 #define HEXDUMP(name, where, size) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
  41                         where, size)
  42 #define HEXDUMP_OFF(name, where, size, idx) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
  43                         &where[idx * size], size)
  44
  45 #define HEXDUMP_OFF_F(name, idx) QAT_DP_HEXDUMP_LOG(DEBUG, name, \
  46                         &cookie->input_buffer[idx * qat_func_alignsize], \
  47                         qat_func_alignsize)
  48 #else
  49 #define HEXDUMP(name, where, size)
  50 #define HEXDUMP_OFF(name, where, size, idx)
  51 #define HEXDUMP_OFF_F(name, idx)
  52 #endif
  53
  54 #define CHECK_IF_NOT_EMPTY(param, name, pname, status) \
  55         do { \
  56                 if (param.length == 0) {        \
  57                         QAT_LOG(ERR,                    \
  58                                 "Invalid " name \
  59                                 " input parameter, zero length " pname  \
  60                         );      \
  61                         status = -EINVAL;       \
  62                 } else if (check_zero(param)) { \
  63                         QAT_LOG(ERR,    \
  64                                 "Invalid " name " input parameter, empty " \
  65                                 pname ", length = %d", \
  66                                 (int)param.length \
  67                         ); \
  68                         status = -EINVAL;       \
  69                 } \
  70         } while (0)
  71
  72 #define SET_PKE_LN(what, how, idx) \
  73                 rte_memcpy(cookie->input_array[idx] + how - \
  74                         what.length, \
  75                         what.data, \
  76                         what.length)
  77
  78 #define SET_PKE_LN_EC(curve, p, idx) \
  79                 rte_memcpy(cookie->input_array[idx] + \
  80                         qat_func_alignsize - curve.bytesize, \
  81                         curve.p.data, curve.bytesize)
  82
  83 #define SET_PKE_9A_IN(what, idx) \
  84                 rte_memcpy(&cookie->input_buffer[idx * \
  85                         qat_func_alignsize] + \
  86                         qat_func_alignsize - what.length, \
  87                         what.data, what.length)
  88
  89 #define SET_PKE_9A_EC(curve, p, idx) \
  90                 rte_memcpy(&cookie->input_buffer[idx * \
  91                         qat_func_alignsize] + \
  92                         qat_func_alignsize - curve.bytesize, \
  93                         curve.p.data, curve.bytesize)
  94
  95 static void
  96 request_init(struct icp_qat_fw_pke_request *qat_req)
  97 {
  98         memset(qat_req, 0, sizeof(*qat_req));
  99         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
 100         qat_req->pke_hdr.hdr_flags =
 101                         ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
 102                         (ICP_QAT_FW_COMN_REQ_FLAG_SET);
 103 }
 104
 105 static void
 106 cleanup_arrays(struct qat_asym_op_cookie *cookie,
 107                 int in_count, int out_count, int alg_size)
 108 {
 109         int i;
 110
 111         for (i = 0; i < in_count; i++)
 112                 memset(cookie->input_array[i], 0x0, alg_size);
 113         for (i = 0; i < out_count; i++)
 114                 memset(cookie->output_array[i], 0x0, alg_size);
 115 }
 116
 117 static void
 118 cleanup_crt(struct qat_asym_op_cookie *cookie,
 119                 int alg_size)
 120 {
 121         int i;
 122
 123         memset(cookie->input_array[0], 0x0, alg_size);
 124         for (i = 1; i < QAT_ASYM_RSA_QT_NUM_IN_PARAMS; i++)
 125                 memset(cookie->input_array[i], 0x0, alg_size / 2);
 126         for (i = 0; i < QAT_ASYM_RSA_NUM_OUT_PARAMS; i++)
 127                 memset(cookie->output_array[i], 0x0, alg_size);
 128 }
 129
 130 static void
 131 cleanup(struct qat_asym_op_cookie *cookie,
 132                 struct rte_crypto_asym_xform *xform, int alg_size)
 133 {
 134         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX)
 135                 cleanup_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
 136                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, alg_size);
 137         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV)
 138                 cleanup_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
 139                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, alg_size);
 140         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 141                 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT)
 142                         cleanup_crt(cookie, alg_size);
 143                 else {
 144                         cleanup_arrays(cookie, QAT_ASYM_RSA_NUM_IN_PARAMS,
 145                                 QAT_ASYM_RSA_NUM_OUT_PARAMS, alg_size);
 146                 }
 147         }
 148 }
 149
 150 static int
 151 check_zero(rte_crypto_param n)
 152 {
 153         int i, len = n.length;
 154
 155         if (len < 8) {
 156                 for (i = len - 1; i >= 0; i--) {
 157                         if (n.data[i] != 0x0)
 158                                 return 0;
 159                 }
 160         } else if (len == 8 && *(uint64_t *)&n.data[len - 8] == 0) {
 161                 return 1;
 162         } else if (*(uint64_t *)&n.data[len - 8] == 0) {
 163                 for (i = len - 9; i >= 0; i--) {
 164                         if (n.data[i] != 0x0)
 165                                 return 0;
 166                 }
 167         } else
 168                 return 0;
 169
 170         return 1;
 171 }
 172
 173 static struct qat_asym_function
 174 get_asym_function(struct rte_crypto_asym_xform *xform)
 175 {
 176         struct qat_asym_function qat_function;
 177
 178         switch (xform->xform_type) {
 179         case RTE_CRYPTO_ASYM_XFORM_MODEX:
 180                 qat_function = get_modexp_function(xform);
 181                 break;
 182         case RTE_CRYPTO_ASYM_XFORM_MODINV:
 183                 qat_function = get_modinv_function(xform);
 184                 break;
 185         default:
 186                 qat_function.func_id = 0;
 187                 break;
 188         }
 189
 190         return qat_function;
 191 }
 192
 193 static int
 194 modexp_set_input(struct rte_crypto_asym_op *asym_op,
 195                 struct icp_qat_fw_pke_request *qat_req,
 196                 struct qat_asym_op_cookie *cookie,
 197                 struct rte_crypto_asym_xform *xform)
 198 {
 199         struct qat_asym_function qat_function;
 200         uint32_t alg_bytesize, func_id, in_bytesize;
 201         int status = 0;
 202
 203         CHECK_IF_NOT_EMPTY(xform->modex.modulus, "mod exp",
 204                         "modulus", status);
 205         CHECK_IF_NOT_EMPTY(xform->modex.exponent, "mod exp",
 206                                 "exponent", status);
 207         if (status)
 208                 return status;
 209
 210         if (asym_op->modex.base.length > xform->modex.exponent.length &&
 211                 asym_op->modex.base.length > xform->modex.modulus.length) {
 212                 in_bytesize = asym_op->modex.base.length;
 213         } else if (xform->modex.exponent.length > xform->modex.modulus.length)
 214                 in_bytesize = xform->modex.exponent.length;
 215         else
 216                 in_bytesize = xform->modex.modulus.length;
 217
 218         qat_function = get_modexp_function2(in_bytesize);
 219         func_id = qat_function.func_id;
 220         if (qat_function.func_id == 0) {
 221                 QAT_LOG(ERR, "Cannot obtain functionality id");
 222                 return -EINVAL;
 223         }
 224         alg_bytesize = qat_function.bytesize;
 225
 226         SET_PKE_LN(asym_op->modex.base, alg_bytesize, 0);
 227         SET_PKE_LN(xform->modex.exponent, alg_bytesize, 1);
 228         SET_PKE_LN(xform->modex.modulus, alg_bytesize, 2);
 229
 230         cookie->alg_bytesize = alg_bytesize;
 231         qat_req->pke_hdr.cd_pars.func_id = func_id;
 232         qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
 233         qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
 234
 235         HEXDUMP("ModExp base", cookie->input_array[0], alg_bytesize);
 236         HEXDUMP("ModExp exponent", cookie->input_array[1], alg_bytesize);
 237         HEXDUMP("ModExp modulus", cookie->input_array[2], alg_bytesize);
 238
 239         return status;
 240 }
 241
 242 static uint8_t
 243 modexp_collect(struct rte_crypto_asym_op *asym_op,
 244                 struct qat_asym_op_cookie *cookie,
 245                 struct rte_crypto_asym_xform *xform)
 246 {
 247         rte_crypto_param n = xform->modex.modulus;
 248         uint32_t alg_bytesize = cookie->alg_bytesize;
 249         uint8_t *modexp_result = asym_op->modex.result.data;
 250
 251         rte_memcpy(modexp_result,
 252                 cookie->output_array[0] + alg_bytesize
 253                 - n.length, n.length);
 254         HEXDUMP("ModExp result", cookie->output_array[0],
 255                         alg_bytesize);
 256         return RTE_CRYPTO_OP_STATUS_SUCCESS;
 257 }
 258
 259 static int
 260 modinv_set_input(struct rte_crypto_asym_op *asym_op,
 261                 struct icp_qat_fw_pke_request *qat_req,
 262                 struct qat_asym_op_cookie *cookie,
 263                 struct rte_crypto_asym_xform *xform)
 264 {
 265         struct qat_asym_function qat_function;
 266         uint32_t alg_bytesize, func_id;
 267         int status = 0;
 268
 269         CHECK_IF_NOT_EMPTY(xform->modex.modulus, "mod inv",
 270                         "modulus", status);
 271         if (status)
 272                 return status;
 273
 274         qat_function = get_asym_function(xform);
 275         func_id = qat_function.func_id;
 276         if (func_id == 0) {
 277                 QAT_LOG(ERR, "Cannot obtain functionality id");
 278                 return -EINVAL;
 279         }
 280         alg_bytesize = qat_function.bytesize;
 281
 282         SET_PKE_LN(asym_op->modinv.base, alg_bytesize, 0);
 283         SET_PKE_LN(xform->modinv.modulus, alg_bytesize, 1);
 284
 285         cookie->alg_bytesize = alg_bytesize;
 286         qat_req->pke_hdr.cd_pars.func_id = func_id;
 287         qat_req->input_param_count =
 288                         QAT_ASYM_MODINV_NUM_IN_PARAMS;
 289         qat_req->output_param_count =
 290                         QAT_ASYM_MODINV_NUM_OUT_PARAMS;
 291
 292         HEXDUMP("ModInv base", cookie->input_array[0], alg_bytesize);
 293         HEXDUMP("ModInv modulus", cookie->input_array[1], alg_bytesize);
 294
 295         return 0;
 296 }
 297
 298 static uint8_t
 299 modinv_collect(struct rte_crypto_asym_op *asym_op,
 300                 struct qat_asym_op_cookie *cookie,
 301                 struct rte_crypto_asym_xform *xform)
 302 {
 303         rte_crypto_param n = xform->modinv.modulus;
 304         uint8_t *modinv_result = asym_op->modinv.result.data;
 305         uint32_t alg_bytesize = cookie->alg_bytesize;
 306
 307         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 308                 - n.length),
 309                 cookie->output_array[0] + alg_bytesize
 310                 - n.length, n.length);
 311         HEXDUMP("ModInv result", cookie->output_array[0],
 312                         alg_bytesize);
 313         return RTE_CRYPTO_OP_STATUS_SUCCESS;
 314 }
 315
 316 static int
 317 rsa_set_pub_input(struct rte_crypto_asym_op *asym_op,
 318                 struct icp_qat_fw_pke_request *qat_req,
 319                 struct qat_asym_op_cookie *cookie,
 320                 struct rte_crypto_asym_xform *xform)
 321 {
 322         struct qat_asym_function qat_function;
 323         uint32_t alg_bytesize, func_id;
 324         int status = 0;
 325
 326         qat_function = get_rsa_enc_function(xform);
 327         func_id = qat_function.func_id;
 328         if (func_id == 0) {
 329                 QAT_LOG(ERR, "Cannot obtain functionality id");
 330                 return -EINVAL;
 331         }
 332         alg_bytesize = qat_function.bytesize;
 333
 334         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 335                 switch (asym_op->rsa.pad) {
 336                 case RTE_CRYPTO_RSA_PADDING_NONE:
 337                         SET_PKE_LN(asym_op->rsa.message, alg_bytesize, 0);
 338                         break;
 339                 default:
 340                         QAT_LOG(ERR,
 341                                 "Invalid RSA padding (Encryption)"
 342                                 );
 343                         return -EINVAL;
 344                 }
 345                 HEXDUMP("RSA Message", cookie->input_array[0], alg_bytesize);
 346         } else {
 347                 switch (asym_op->rsa.pad) {
 348                 case RTE_CRYPTO_RSA_PADDING_NONE:
 349                         SET_PKE_LN(asym_op->rsa.sign, alg_bytesize, 0);
 350                         break;
 351                 default:
 352                         QAT_LOG(ERR,
 353                                 "Invalid RSA padding (Verify)");
 354                         return -EINVAL;
 355                 }
 356                 HEXDUMP("RSA Signature", cookie->input_array[0],
 357                                 alg_bytesize);
 358         }
 359
 360         SET_PKE_LN(xform->rsa.e, alg_bytesize, 1);
 361         SET_PKE_LN(xform->rsa.n, alg_bytesize, 2);
 362
 363         cookie->alg_bytesize = alg_bytesize;
 364         qat_req->pke_hdr.cd_pars.func_id = func_id;
 365
 366         HEXDUMP("RSA Public Key", cookie->input_array[1], alg_bytesize);
 367         HEXDUMP("RSA Modulus", cookie->input_array[2], alg_bytesize);
 368
 369         return status;
 370 }
 371
 372 static int
 373 rsa_set_priv_input(struct rte_crypto_asym_op *asym_op,
 374                 struct icp_qat_fw_pke_request *qat_req,
 375                 struct qat_asym_op_cookie *cookie,
 376                 struct rte_crypto_asym_xform *xform)
 377 {
 378         struct qat_asym_function qat_function;
 379         uint32_t alg_bytesize, func_id;
 380         int status = 0;
 381
 382         if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT) {
 383                 qat_function = get_rsa_crt_function(xform);
 384                 func_id = qat_function.func_id;
 385                 if (func_id == 0) {
 386                         QAT_LOG(ERR, "Cannot obtain functionality id");
 387                         return -EINVAL;
 388                 }
 389                 alg_bytesize = qat_function.bytesize;
 390                 qat_req->input_param_count =
 391                                 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
 392
 393                 SET_PKE_LN(xform->rsa.qt.p, (alg_bytesize >> 1), 1);
 394                 SET_PKE_LN(xform->rsa.qt.q, (alg_bytesize >> 1), 2);
 395                 SET_PKE_LN(xform->rsa.qt.dP, (alg_bytesize >> 1), 3);
 396                 SET_PKE_LN(xform->rsa.qt.dQ, (alg_bytesize >> 1), 4);
 397                 SET_PKE_LN(xform->rsa.qt.qInv, (alg_bytesize >> 1), 5);
 398
 399                 HEXDUMP("RSA p", cookie->input_array[1],
 400                                 alg_bytesize);
 401                 HEXDUMP("RSA q", cookie->input_array[2],
 402                                 alg_bytesize);
 403                 HEXDUMP("RSA dP", cookie->input_array[3],
 404                                 alg_bytesize);
 405                 HEXDUMP("RSA dQ", cookie->input_array[4],
 406                                 alg_bytesize);
 407                 HEXDUMP("RSA qInv", cookie->input_array[5],
 408                                 alg_bytesize);
 409         } else if (xform->rsa.key_type ==
 410                         RTE_RSA_KEY_TYPE_EXP) {
 411                 qat_function = get_rsa_dec_function(xform);
 412                 func_id = qat_function.func_id;
 413                 if (func_id == 0) {
 414                         QAT_LOG(ERR, "Cannot obtain functionality id");
 415                         return -EINVAL;
 416                 }
 417                 alg_bytesize = qat_function.bytesize;
 418
 419                 SET_PKE_LN(xform->rsa.d, alg_bytesize, 1);
 420                 SET_PKE_LN(xform->rsa.n, alg_bytesize, 2);
 421
 422                 HEXDUMP("RSA d", cookie->input_array[1],
 423                                 alg_bytesize);
 424                 HEXDUMP("RSA n", cookie->input_array[2],
 425                                 alg_bytesize);
 426         } else {
 427                 QAT_LOG(ERR, "Invalid RSA key type");
 428                 return -EINVAL;
 429         }
 430
 431         if (asym_op->rsa.op_type ==
 432                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 433                 switch (asym_op->rsa.pad) {
 434                 case RTE_CRYPTO_RSA_PADDING_NONE:
 435                         SET_PKE_LN(asym_op->rsa.cipher, alg_bytesize, 0);
 436                         HEXDUMP("RSA ciphertext", cookie->input_array[0],
 437                                 alg_bytesize);
 438                         break;
 439                 default:
 440                         QAT_LOG(ERR,
 441                                 "Invalid padding of RSA (Decrypt)");
 442                         return -(EINVAL);
 443                 }
 444
 445         } else if (asym_op->rsa.op_type ==
 446                         RTE_CRYPTO_ASYM_OP_SIGN) {
 447                 switch (asym_op->rsa.pad) {
 448                 case RTE_CRYPTO_RSA_PADDING_NONE:
 449                         SET_PKE_LN(asym_op->rsa.message, alg_bytesize, 0);
 450                         HEXDUMP("RSA text to be signed", cookie->input_array[0],
 451                                 alg_bytesize);
 452                         break;
 453                 default:
 454                         QAT_LOG(ERR,
 455                                 "Invalid padding of RSA (Signature)");
 456                         return -(EINVAL);
 457                 }
 458         }
 459
 460         cookie->alg_bytesize = alg_bytesize;
 461         qat_req->pke_hdr.cd_pars.func_id = func_id;
 462         return status;
 463 }
 464
 465 static int
 466 rsa_set_input(struct rte_crypto_asym_op *asym_op,
 467                 struct icp_qat_fw_pke_request *qat_req,
 468                 struct qat_asym_op_cookie *cookie,
 469                 struct rte_crypto_asym_xform *xform)
 470 {
 471         qat_req->input_param_count =
 472                         QAT_ASYM_RSA_NUM_IN_PARAMS;
 473         qat_req->output_param_count =
 474                         QAT_ASYM_RSA_NUM_OUT_PARAMS;
 475
 476         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 477                         asym_op->rsa.op_type ==
 478                                 RTE_CRYPTO_ASYM_OP_VERIFY) {
 479                 return rsa_set_pub_input(asym_op, qat_req, cookie, xform);
 480         } else {
 481                 return rsa_set_priv_input(asym_op, qat_req, cookie, xform);
 482         }
 483 }
 484
 485 static uint8_t
 486 rsa_collect(struct rte_crypto_asym_op *asym_op,
 487                 struct qat_asym_op_cookie *cookie)
 488 {
 489         uint32_t alg_bytesize = cookie->alg_bytesize;
 490
 491         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 492                 asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_VERIFY) {
 493
 494                 if (asym_op->rsa.op_type ==
 495                                 RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 496                         uint8_t *rsa_result = asym_op->rsa.cipher.data;
 497
 498                         rte_memcpy(rsa_result,
 499                                         cookie->output_array[0],
 500                                         alg_bytesize);
 501                         HEXDUMP("RSA Encrypted data", cookie->output_array[0],
 502                                 alg_bytesize);
 503                 } else {
 504                         uint8_t *rsa_result = asym_op->rsa.cipher.data;
 505
 506                         switch (asym_op->rsa.pad) {
 507                         case RTE_CRYPTO_RSA_PADDING_NONE:
 508                                 rte_memcpy(rsa_result,
 509                                                 cookie->output_array[0],
 510                                                 alg_bytesize);
 511                                 HEXDUMP("RSA signature",
 512                                         cookie->output_array[0],
 513                                         alg_bytesize);
 514                                 break;
 515                         default:
 516                                 QAT_LOG(ERR, "Padding not supported");
 517                                 return RTE_CRYPTO_OP_STATUS_ERROR;
 518                         }
 519                 }
 520         } else {
 521                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_DECRYPT) {
 522                         uint8_t *rsa_result = asym_op->rsa.message.data;
 523
 524                         switch (asym_op->rsa.pad) {
 525                         case RTE_CRYPTO_RSA_PADDING_NONE:
 526                                 rte_memcpy(rsa_result,
 527                                         cookie->output_array[0],
 528                                         alg_bytesize);
 529                                 HEXDUMP("RSA Decrypted Message",
 530                                         cookie->output_array[0],
 531                                         alg_bytesize);
 532                                 break;
 533                         default:
 534                                 QAT_LOG(ERR, "Padding not supported");
 535                                 return RTE_CRYPTO_OP_STATUS_ERROR;
 536                         }
 537                 } else {
 538                         uint8_t *rsa_result = asym_op->rsa.sign.data;
 539
 540                         rte_memcpy(rsa_result,
 541                                         cookie->output_array[0],
 542                                         alg_bytesize);
 543                         HEXDUMP("RSA Signature", cookie->output_array[0],
 544                                 alg_bytesize);
 545                 }
 546         }
 547         return RTE_CRYPTO_OP_STATUS_SUCCESS;
 548 }
 549
 550 static int
 551 ecdsa_set_input(struct rte_crypto_asym_op *asym_op,
 552                 struct icp_qat_fw_pke_request *qat_req,
 553                 struct qat_asym_op_cookie *cookie,
 554                 struct rte_crypto_asym_xform *xform)
 555 {
 556         struct qat_asym_function qat_function;
 557         uint32_t qat_func_alignsize, func_id;
 558         int curve_id;
 559
 560         curve_id = pick_curve(xform);
 561         if (curve_id < 0) {
 562                 QAT_LOG(DEBUG, "Incorrect elliptic curve");
 563                 return -EINVAL;
 564         }
 565
 566         switch (asym_op->ecdsa.op_type) {
 567         case RTE_CRYPTO_ASYM_OP_SIGN:
 568                 qat_function = get_ecdsa_function(xform);
 569                 func_id = qat_function.func_id;
 570                 if (func_id == 0) {
 571                         QAT_LOG(ERR, "Cannot obtain functionality id");
 572                         return -EINVAL;
 573                 }
 574                 qat_func_alignsize =
 575                         RTE_ALIGN_CEIL(qat_function.bytesize, 8);
 576
 577                 SET_PKE_9A_IN(asym_op->ecdsa.pkey, 0);
 578                 SET_PKE_9A_IN(asym_op->ecdsa.message, 1);
 579                 SET_PKE_9A_IN(asym_op->ecdsa.k, 2);
 580                 SET_PKE_9A_EC(curve[curve_id], b, 3);
 581                 SET_PKE_9A_EC(curve[curve_id], a, 4);
 582                 SET_PKE_9A_EC(curve[curve_id], p, 5);
 583                 SET_PKE_9A_EC(curve[curve_id], n, 6);
 584                 SET_PKE_9A_EC(curve[curve_id], y, 7);
 585                 SET_PKE_9A_EC(curve[curve_id], x, 8);
 586
 587                 cookie->alg_bytesize = curve[curve_id].bytesize;
 588                 cookie->qat_func_alignsize = qat_func_alignsize;
 589                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 590                 qat_req->input_param_count =
 591                                 QAT_ASYM_ECDSA_RS_SIGN_IN_PARAMS;
 592                 qat_req->output_param_count =
 593                                 QAT_ASYM_ECDSA_RS_SIGN_OUT_PARAMS;
 594
 595                 HEXDUMP_OFF_F("ECDSA d", 0);
 596                 HEXDUMP_OFF_F("ECDSA e", 1);
 597                 HEXDUMP_OFF_F("ECDSA k", 2);
 598                 HEXDUMP_OFF_F("ECDSA b", 3);
 599                 HEXDUMP_OFF_F("ECDSA a", 4);
 600                 HEXDUMP_OFF_F("ECDSA n", 5);
 601                 HEXDUMP_OFF_F("ECDSA y", 6);
 602                 HEXDUMP_OFF_F("ECDSA x", 7);
 603                 break;
 604         case RTE_CRYPTO_ASYM_OP_VERIFY:
 605                 qat_function = get_ecdsa_verify_function(xform);
 606                 func_id = qat_function.func_id;
 607                 if (func_id == 0) {
 608                         QAT_LOG(ERR, "Cannot obtain functionality id");
 609                         return -EINVAL;
 610                 }
 611                 qat_func_alignsize = RTE_ALIGN_CEIL(qat_function.bytesize, 8);
 612
 613                 SET_PKE_9A_IN(asym_op->ecdsa.message, 10);
 614                 SET_PKE_9A_IN(asym_op->ecdsa.s, 9);
 615                 SET_PKE_9A_IN(asym_op->ecdsa.r, 8);
 616                 SET_PKE_9A_EC(curve[curve_id], n, 7);
 617                 SET_PKE_9A_EC(curve[curve_id], x, 6);
 618                 SET_PKE_9A_EC(curve[curve_id], y, 5);
 619                 SET_PKE_9A_IN(asym_op->ecdsa.q.x, 4);
 620                 SET_PKE_9A_IN(asym_op->ecdsa.q.y, 3);
 621                 SET_PKE_9A_EC(curve[curve_id], a, 2);
 622                 SET_PKE_9A_EC(curve[curve_id], b, 1);
 623                 SET_PKE_9A_EC(curve[curve_id], p, 0);
 624
 625                 cookie->alg_bytesize = curve[curve_id].bytesize;
 626                 cookie->qat_func_alignsize = qat_func_alignsize;
 627                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 628                 qat_req->input_param_count =
 629                                 QAT_ASYM_ECDSA_RS_VERIFY_IN_PARAMS;
 630                 qat_req->output_param_count =
 631                                 QAT_ASYM_ECDSA_RS_VERIFY_OUT_PARAMS;
 632
 633                 HEXDUMP_OFF_F("p", 0);
 634                 HEXDUMP_OFF_F("b", 1);
 635                 HEXDUMP_OFF_F("a", 2);
 636                 HEXDUMP_OFF_F("y", 3);
 637                 HEXDUMP_OFF_F("x", 4);
 638                 HEXDUMP_OFF_F("yG", 5);
 639                 HEXDUMP_OFF_F("xG", 6);
 640                 HEXDUMP_OFF_F("n", 7);
 641                 HEXDUMP_OFF_F("r", 8);
 642                 HEXDUMP_OFF_F("s", 9);
 643                 HEXDUMP_OFF_F("e", 10);
 644                 break;
 645         default:
 646                 return -1;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static uint8_t
 653 ecdsa_collect(struct rte_crypto_asym_op *asym_op,
 654                 struct qat_asym_op_cookie *cookie)
 655 {
 656         uint32_t alg_bytesize = cookie->alg_bytesize;
 657         uint32_t qat_func_alignsize = cookie->qat_func_alignsize;
 658         uint32_t ltrim = qat_func_alignsize - alg_bytesize;
 659
 660         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
 661                 uint8_t *r = asym_op->ecdsa.r.data;
 662                 uint8_t *s = asym_op->ecdsa.s.data;
 663
 664                 asym_op->ecdsa.r.length = alg_bytesize;
 665                 asym_op->ecdsa.s.length = alg_bytesize;
 666                 rte_memcpy(r, &cookie->output_array[0][ltrim], alg_bytesize);
 667                 rte_memcpy(s, &cookie->output_array[1][ltrim], alg_bytesize);
 668
 669                 HEXDUMP("R", cookie->output_array[0],
 670                         qat_func_alignsize);
 671                 HEXDUMP("S", cookie->output_array[1],
 672                         qat_func_alignsize);
 673         }
 674         return RTE_CRYPTO_OP_STATUS_SUCCESS;
 675 }
 676
 677 static int
 678 ecpm_set_input(struct rte_crypto_asym_op *asym_op,
 679                 struct icp_qat_fw_pke_request *qat_req,
 680                 struct qat_asym_op_cookie *cookie,
 681                 struct rte_crypto_asym_xform *xform)
 682 {
 683         struct qat_asym_function qat_function;
 684         uint32_t qat_func_alignsize, func_id;
 685         int curve_id;
 686
 687         curve_id = pick_curve(xform);
 688         if (curve_id < 0) {
 689                 QAT_LOG(DEBUG, "Incorrect elliptic curve");
 690                 return -EINVAL;
 691         }
 692
 693         qat_function = get_ecpm_function(xform);
 694         func_id = qat_function.func_id;
 695         if (func_id == 0) {
 696                 QAT_LOG(ERR, "Cannot obtain functionality id");
 697                 return -EINVAL;
 698         }
 699         qat_func_alignsize = RTE_ALIGN_CEIL(qat_function.bytesize, 8);
 700
 701         SET_PKE_LN(asym_op->ecpm.scalar, qat_func_alignsize, 0);
 702         SET_PKE_LN(asym_op->ecpm.p.x, qat_func_alignsize, 1);
 703         SET_PKE_LN(asym_op->ecpm.p.y, qat_func_alignsize, 2);
 704         SET_PKE_LN_EC(curve[curve_id], a, 3);
 705         SET_PKE_LN_EC(curve[curve_id], b, 4);
 706         SET_PKE_LN_EC(curve[curve_id], p, 5);
 707         SET_PKE_LN_EC(curve[curve_id], h, 6);
 708
 709         cookie->alg_bytesize = curve[curve_id].bytesize;
 710         cookie->qat_func_alignsize = qat_func_alignsize;
 711         qat_req->pke_hdr.cd_pars.func_id = func_id;
 712         qat_req->input_param_count =
 713                         QAT_ASYM_ECPM_IN_PARAMS;
 714         qat_req->output_param_count =
 715                         QAT_ASYM_ECPM_OUT_PARAMS;
 716
 717         HEXDUMP("k", cookie->input_array[0], qat_func_alignsize);
 718         HEXDUMP("xG", cookie->input_array[1], qat_func_alignsize);
 719         HEXDUMP("yG", cookie->input_array[2], qat_func_alignsize);
 720         HEXDUMP("a", cookie->input_array[3], qat_func_alignsize);
 721         HEXDUMP("b", cookie->input_array[4], qat_func_alignsize);
 722         HEXDUMP("q", cookie->input_array[5], qat_func_alignsize);
 723         HEXDUMP("h", cookie->input_array[6], qat_func_alignsize);
 724
 725         return 0;
 726 }
 727
 728 static uint8_t
 729 ecpm_collect(struct rte_crypto_asym_op *asym_op,
 730                 struct qat_asym_op_cookie *cookie)
 731 {
 732         uint8_t *x = asym_op->ecpm.r.x.data;
 733         uint8_t *y = asym_op->ecpm.r.y.data;
 734         uint32_t alg_bytesize = cookie->alg_bytesize;
 735         uint32_t qat_func_alignsize = cookie->qat_func_alignsize;
 736         uint32_t ltrim = qat_func_alignsize - alg_bytesize;
 737
 738         asym_op->ecpm.r.x.length = alg_bytesize;
 739         asym_op->ecpm.r.y.length = alg_bytesize;
 740         rte_memcpy(x, &cookie->output_array[0][ltrim], alg_bytesize);
 741         rte_memcpy(y, &cookie->output_array[1][ltrim], alg_bytesize);
 742
 743         HEXDUMP("rX", cookie->output_array[0],
 744                 qat_func_alignsize);
 745         HEXDUMP("rY", cookie->output_array[1],
 746                 qat_func_alignsize);
 747         return RTE_CRYPTO_OP_STATUS_SUCCESS;
 748 }
 749
 750 static int
 751 asym_set_input(struct rte_crypto_asym_op *asym_op,
 752                 struct icp_qat_fw_pke_request *qat_req,
 753                 struct qat_asym_op_cookie *cookie,
 754                 struct rte_crypto_asym_xform *xform)
 755 {
 756         switch (xform->xform_type) {
 757         case RTE_CRYPTO_ASYM_XFORM_MODEX:
 758                 return modexp_set_input(asym_op, qat_req,
 759                                 cookie, xform);
 760         case RTE_CRYPTO_ASYM_XFORM_MODINV:
 761                 return modinv_set_input(asym_op, qat_req,
 762                                 cookie, xform);
 763         case RTE_CRYPTO_ASYM_XFORM_RSA:
 764                 return rsa_set_input(asym_op, qat_req,
 765                                 cookie, xform);
 766         case RTE_CRYPTO_ASYM_XFORM_ECDSA:
 767                 return ecdsa_set_input(asym_op, qat_req,
 768                                 cookie, xform);
 769         case RTE_CRYPTO_ASYM_XFORM_ECPM:
 770                 return ecpm_set_input(asym_op, qat_req,
 771                                 cookie, xform);
 772         default:
 773                 QAT_LOG(ERR, "Invalid/unsupported asymmetric crypto xform");
 774                 return -EINVAL;
 775         }
 776         return 1;
 777 }
 778
 779 static int
 780 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 781                         __rte_unused uint64_t *opaque,
 782                         __rte_unused enum qat_device_gen qat_dev_gen)
 783 {
 784         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 785         struct rte_crypto_asym_op *asym_op = op->asym;
 786         struct icp_qat_fw_pke_request *qat_req =
 787                         (struct icp_qat_fw_pke_request *)out_msg;
 788         struct qat_asym_op_cookie *cookie =
 789                         (struct qat_asym_op_cookie *)op_cookie;
 790         struct rte_crypto_asym_xform *xform;
 791         struct qat_asym_session *qat_session = (struct qat_asym_session *)
 792                         op->asym->session->sess_private_data;
 793         int err = 0;
 794
 795         if (unlikely(qat_session == NULL)) {
 796                 QAT_DP_LOG(ERR, "Session was not created for this device");
 797                 goto error;
 798         }
 799
 800         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 801         switch (op->sess_type) {
 802         case RTE_CRYPTO_OP_WITH_SESSION:
 803                 request_init(qat_req);
 804                 xform = &qat_session->xform;
 805                 break;
 806         case RTE_CRYPTO_OP_SESSIONLESS:
 807                 request_init(qat_req);
 808                 xform = op->asym->xform;
 809                 break;
 810         default:
 811                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
 812                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 813                 goto error;
 814         }
 815         err = asym_set_input(asym_op, qat_req, cookie,
 816                         xform);
 817         if (err) {
 818                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 819                 goto error;
 820         }
 821
 822         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 823         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 824         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 825
 826         HEXDUMP("qat_req:", qat_req, sizeof(struct icp_qat_fw_pke_request));
 827
 828         return 0;
 829 error:
 830         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 831         HEXDUMP("qat_req:", qat_req, sizeof(struct icp_qat_fw_pke_request));
 832         qat_req->output_param_count = 0;
 833         qat_req->input_param_count = 0;
 834         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 835         cookie->error |= err;
 836
 837         return 0;
 838 }
 839
 840 static uint8_t
 841 qat_asym_collect_response(struct rte_crypto_op *op,
 842                 struct qat_asym_op_cookie *cookie,
 843                 struct rte_crypto_asym_xform *xform)
 844 {
 845         struct rte_crypto_asym_op *asym_op = op->asym;
 846
 847         switch (xform->xform_type) {
 848         case RTE_CRYPTO_ASYM_XFORM_MODEX:
 849                 return modexp_collect(asym_op, cookie, xform);
 850         case RTE_CRYPTO_ASYM_XFORM_MODINV:
 851                 return modinv_collect(asym_op, cookie, xform);
 852         case RTE_CRYPTO_ASYM_XFORM_RSA:
 853                 return rsa_collect(asym_op, cookie);
 854         case RTE_CRYPTO_ASYM_XFORM_ECDSA:
 855                 return ecdsa_collect(asym_op, cookie);
 856         case RTE_CRYPTO_ASYM_XFORM_ECPM:
 857                 return ecpm_collect(asym_op, cookie);
 858         default:
 859                 QAT_LOG(ERR, "Not supported xform type");
 860                 return  RTE_CRYPTO_OP_STATUS_ERROR;
 861         }
 862 }
 863
 864 static int
 865 qat_asym_process_response(void **out_op, uint8_t *resp,
 866                 void *op_cookie, __rte_unused uint64_t *dequeue_err_count)
 867 {
 868         struct icp_qat_fw_pke_resp *resp_msg =
 869                         (struct icp_qat_fw_pke_resp *)resp;
 870         struct rte_crypto_op *op = (struct rte_crypto_op *)(uintptr_t)
 871                         (resp_msg->opaque);
 872         struct qat_asym_op_cookie *cookie = op_cookie;
 873         struct rte_crypto_asym_xform *xform;
 874         struct qat_asym_session *qat_session = (struct qat_asym_session *)
 875                         op->asym->session->sess_private_data;
 876
 877         if (cookie->error) {
 878                 cookie->error = 0;
 879                 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 880                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 881                 QAT_DP_LOG(DEBUG, "Cookie status returned error");
 882         } else {
 883                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 884                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 885                         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 886                                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 887                         QAT_DP_LOG(DEBUG, "Asymmetric response status"
 888                                         " returned error");
 889                 }
 890                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 891                         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 892                                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 893                         QAT_DP_LOG(ERR, "Asymmetric common status"
 894                                         " returned error");
 895                 }
 896         }
 897
 898         switch (op->sess_type) {
 899         case RTE_CRYPTO_OP_WITH_SESSION:
 900                 xform = &qat_session->xform;
 901                 break;
 902         case RTE_CRYPTO_OP_SESSIONLESS:
 903                 xform = op->asym->xform;
 904                 break;
 905         default:
 906                 QAT_DP_LOG(ERR,
 907                         "Invalid session/xform settings in response ring!");
 908                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 909         }
 910
 911         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 912                 op->status = qat_asym_collect_response(op,
 913                                         cookie, xform);
 914                 cleanup(cookie, xform, cookie->alg_bytesize);
 915         }
 916
 917         *out_op = op;
 918         HEXDUMP("resp_msg:", resp_msg, sizeof(struct icp_qat_fw_pke_resp));
 919
 920         return 1;
 921 }
 922
 923 static int
 924 session_set_modexp(struct qat_asym_session *qat_session,
 925                         struct rte_crypto_asym_xform *xform)
 926 {
 927         uint8_t *modulus = xform->modex.modulus.data;
 928         uint8_t *exponent = xform->modex.exponent.data;
 929
 930         qat_session->xform.modex.modulus.data =
 931                 rte_malloc(NULL, xform->modex.modulus.length, 0);
 932         if (qat_session->xform.modex.modulus.data == NULL)
 933                 return -ENOMEM;
 934         qat_session->xform.modex.modulus.length = xform->modex.modulus.length;
 935         qat_session->xform.modex.exponent.data = rte_malloc(NULL,
 936                                 xform->modex.exponent.length, 0);
 937         if (qat_session->xform.modex.exponent.data == NULL) {
 938                 rte_free(qat_session->xform.modex.exponent.data);
 939                 return -ENOMEM;
 940         }
 941         qat_session->xform.modex.exponent.length = xform->modex.exponent.length;
 942
 943         rte_memcpy(qat_session->xform.modex.modulus.data, modulus,
 944                         xform->modex.modulus.length);
 945         rte_memcpy(qat_session->xform.modex.exponent.data, exponent,
 946                         xform->modex.exponent.length);
 947
 948         return 0;
 949 }
 950
 951 static int
 952 session_set_modinv(struct qat_asym_session *qat_session,
 953                         struct rte_crypto_asym_xform *xform)
 954 {
 955         uint8_t *modulus = xform->modinv.modulus.data;
 956
 957         qat_session->xform.modinv.modulus.data =
 958                 rte_malloc(NULL, xform->modinv.modulus.length, 0);
 959         if (qat_session->xform.modinv.modulus.data == NULL)
 960                 return -ENOMEM;
 961         qat_session->xform.modinv.modulus.length = xform->modinv.modulus.length;
 962
 963         rte_memcpy(qat_session->xform.modinv.modulus.data, modulus,
 964                         xform->modinv.modulus.length);
 965
 966         return 0;
 967 }
 968
 969 static int
 970 session_set_rsa(struct qat_asym_session *qat_session,
 971                         struct rte_crypto_asym_xform *xform)
 972 {
 973         uint8_t *n = xform->rsa.n.data;
 974         uint8_t *e = xform->rsa.e.data;
 975         int ret = 0;
 976
 977         qat_session->xform.rsa.key_type = xform->rsa.key_type;
 978
 979         qat_session->xform.rsa.n.data =
 980                 rte_malloc(NULL, xform->rsa.n.length, 0);
 981         if (qat_session->xform.rsa.n.data == NULL)
 982                 return -ENOMEM;
 983         qat_session->xform.rsa.n.length =
 984                 xform->rsa.n.length;
 985
 986         qat_session->xform.rsa.e.data =
 987                 rte_malloc(NULL, xform->rsa.e.length, 0);
 988         if (qat_session->xform.rsa.e.data == NULL) {
 989                 ret = -ENOMEM;
 990                 goto err;
 991         }
 992         qat_session->xform.rsa.e.length =
 993                 xform->rsa.e.length;
 994
 995         if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT) {
 996                 uint8_t *p = xform->rsa.qt.p.data;
 997                 uint8_t *q = xform->rsa.qt.q.data;
 998                 uint8_t *dP = xform->rsa.qt.dP.data;
 999                 uint8_t *dQ = xform->rsa.qt.dQ.data;
1000                 uint8_t *qInv = xform->rsa.qt.qInv.data;
1001
1002                 qat_session->xform.rsa.qt.p.data =
1003                         rte_malloc(NULL, xform->rsa.qt.p.length, 0);
1004                 if (qat_session->xform.rsa.qt.p.data == NULL) {
1005                         ret = -ENOMEM;
1006                         goto err;
1007                 }
1008                 qat_session->xform.rsa.qt.p.length =
1009                         xform->rsa.qt.p.length;
1010
1011                 qat_session->xform.rsa.qt.q.data =
1012                         rte_malloc(NULL, xform->rsa.qt.q.length, 0);
1013                 if (qat_session->xform.rsa.qt.q.data == NULL) {
1014                         ret = -ENOMEM;
1015                         goto err;
1016                 }
1017                 qat_session->xform.rsa.qt.q.length =
1018                         xform->rsa.qt.q.length;
1019
1020                 qat_session->xform.rsa.qt.dP.data =
1021                         rte_malloc(NULL, xform->rsa.qt.dP.length, 0);
1022                 if (qat_session->xform.rsa.qt.dP.data == NULL) {
1023                         ret = -ENOMEM;
1024                         goto err;
1025                 }
1026                 qat_session->xform.rsa.qt.dP.length =
1027                         xform->rsa.qt.dP.length;
1028
1029                 qat_session->xform.rsa.qt.dQ.data =
1030                         rte_malloc(NULL, xform->rsa.qt.dQ.length, 0);
1031                 if (qat_session->xform.rsa.qt.dQ.data == NULL) {
1032                         ret = -ENOMEM;
1033                         goto err;
1034                 }
1035                 qat_session->xform.rsa.qt.dQ.length =
1036                         xform->rsa.qt.dQ.length;
1037
1038                 qat_session->xform.rsa.qt.qInv.data =
1039                         rte_malloc(NULL, xform->rsa.qt.qInv.length, 0);
1040                 if (qat_session->xform.rsa.qt.qInv.data == NULL) {
1041                         ret = -ENOMEM;
1042                         goto err;
1043                 }
1044                 qat_session->xform.rsa.qt.qInv.length =
1045                         xform->rsa.qt.qInv.length;
1046
1047                 rte_memcpy(qat_session->xform.rsa.qt.p.data, p,
1048                                 xform->rsa.qt.p.length);
1049                 rte_memcpy(qat_session->xform.rsa.qt.q.data, q,
1050                                 xform->rsa.qt.q.length);
1051                 rte_memcpy(qat_session->xform.rsa.qt.dP.data, dP,
1052                                 xform->rsa.qt.dP.length);
1053                 rte_memcpy(qat_session->xform.rsa.qt.dQ.data, dQ,
1054                                 xform->rsa.qt.dQ.length);
1055                 rte_memcpy(qat_session->xform.rsa.qt.qInv.data, qInv,
1056                                 xform->rsa.qt.qInv.length);
1057
1058         } else {
1059                 uint8_t *d = xform->rsa.d.data;
1060
1061                 qat_session->xform.rsa.d.data =
1062                         rte_malloc(NULL, xform->rsa.d.length, 0);
1063                 if (qat_session->xform.rsa.d.data == NULL) {
1064                         ret = -ENOMEM;
1065                         goto err;
1066                 }
1067                 qat_session->xform.rsa.d.length =
1068                         xform->rsa.d.length;
1069                 rte_memcpy(qat_session->xform.rsa.d.data, d,
1070                         xform->rsa.d.length);
1071         }
1072
1073         rte_memcpy(qat_session->xform.rsa.n.data, n,
1074                 xform->rsa.n.length);
1075         rte_memcpy(qat_session->xform.rsa.e.data, e,
1076                 xform->rsa.e.length);
1077
1078         return 0;
1079
1080 err:
1081         rte_free(qat_session->xform.rsa.n.data);
1082         rte_free(qat_session->xform.rsa.e.data);
1083         rte_free(qat_session->xform.rsa.d.data);
1084         rte_free(qat_session->xform.rsa.qt.p.data);
1085         rte_free(qat_session->xform.rsa.qt.q.data);
1086         rte_free(qat_session->xform.rsa.qt.dP.data);
1087         rte_free(qat_session->xform.rsa.qt.dQ.data);
1088         rte_free(qat_session->xform.rsa.qt.qInv.data);
1089         return ret;
1090 }
1091
1092 static void
1093 session_set_ecdsa(struct qat_asym_session *qat_session,
1094                         struct rte_crypto_asym_xform *xform)
1095 {
1096         qat_session->xform.ec.curve_id = xform->ec.curve_id;
1097 }
1098
1099 int
1100 qat_asym_session_configure(struct rte_cryptodev *dev __rte_unused,
1101                 struct rte_crypto_asym_xform *xform,
1102                 struct rte_cryptodev_asym_session *session)
1103 {
1104         struct qat_asym_session *qat_session;
1105         int ret = 0;
1106
1107         qat_session = (struct qat_asym_session *) session->sess_private_data;
1108         memset(qat_session, 0, sizeof(*qat_session));
1109
1110         qat_session->xform.xform_type = xform->xform_type;
1111         switch (xform->xform_type) {
1112         case RTE_CRYPTO_ASYM_XFORM_MODEX:
1113                 ret = session_set_modexp(qat_session, xform);
1114                 break;
1115         case RTE_CRYPTO_ASYM_XFORM_MODINV:
1116                 ret = session_set_modinv(qat_session, xform);
1117                 break;
1118         case RTE_CRYPTO_ASYM_XFORM_RSA:
1119                 ret = session_set_rsa(qat_session, xform);
1120                 break;
1121         case RTE_CRYPTO_ASYM_XFORM_ECDSA:
1122         case RTE_CRYPTO_ASYM_XFORM_ECPM:
1123                 session_set_ecdsa(qat_session, xform);
1124                 break;
1125         default:
1126                 ret = -ENOTSUP;
1127         }
1128
1129         if (ret) {
1130                 QAT_LOG(ERR, "Unsupported xform type");
1131                 return ret;
1132         }
1133
1134         return 0;
1135 }
1136
1137 unsigned int
1138 qat_asym_session_get_private_size(struct rte_cryptodev *dev __rte_unused)
1139 {
1140         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
1141 }
1142
1143 static void
1144 session_clear_modexp(struct rte_crypto_modex_xform *modex)
1145 {
1146         memset(modex->modulus.data, 0, modex->modulus.length);
1147         rte_free(modex->modulus.data);
1148         memset(modex->exponent.data, 0, modex->exponent.length);
1149         rte_free(modex->exponent.data);
1150 }
1151
1152 static void
1153 session_clear_modinv(struct rte_crypto_modinv_xform *modinv)
1154 {
1155         memset(modinv->modulus.data, 0, modinv->modulus.length);
1156         rte_free(modinv->modulus.data);
1157 }
1158
1159 static void
1160 session_clear_rsa(struct rte_crypto_rsa_xform *rsa)
1161 {
1162         memset(rsa->n.data, 0, rsa->n.length);
1163         rte_free(rsa->n.data);
1164         memset(rsa->e.data, 0, rsa->e.length);
1165         rte_free(rsa->e.data);
1166         if (rsa->key_type == RTE_RSA_KEY_TYPE_EXP) {
1167                 memset(rsa->d.data, 0, rsa->d.length);
1168                 rte_free(rsa->d.data);
1169         } else {
1170                 memset(rsa->qt.p.data, 0, rsa->qt.p.length);
1171                 rte_free(rsa->qt.p.data);
1172                 memset(rsa->qt.q.data, 0, rsa->qt.q.length);
1173                 rte_free(rsa->qt.q.data);
1174                 memset(rsa->qt.dP.data, 0, rsa->qt.dP.length);
1175                 rte_free(rsa->qt.dP.data);
1176                 memset(rsa->qt.dQ.data, 0, rsa->qt.dQ.length);
1177                 rte_free(rsa->qt.dQ.data);
1178                 memset(rsa->qt.qInv.data, 0, rsa->qt.qInv.length);
1179                 rte_free(rsa->qt.qInv.data);
1180         }
1181 }
1182
1183 static void
1184 session_clear_xform(struct qat_asym_session *qat_session)
1185 {
1186         switch (qat_session->xform.xform_type) {
1187         case RTE_CRYPTO_ASYM_XFORM_MODEX:
1188                 session_clear_modexp(&qat_session->xform.modex);
1189                 break;
1190         case RTE_CRYPTO_ASYM_XFORM_MODINV:
1191                 session_clear_modinv(&qat_session->xform.modinv);
1192                 break;
1193         case RTE_CRYPTO_ASYM_XFORM_RSA:
1194                 session_clear_rsa(&qat_session->xform.rsa);
1195                 break;
1196         default:
1197                 break;
1198         }
1199 }
1200
1201 void
1202 qat_asym_session_clear(struct rte_cryptodev *dev,
1203                 struct rte_cryptodev_asym_session *session)
1204 {
1205         void *sess_priv = session->sess_private_data;
1206         struct qat_asym_session *qat_session =
1207                 (struct qat_asym_session *)sess_priv;
1208
1209         if (sess_priv) {
1210                 session_clear_xform(qat_session);
1211                 memset(qat_session, 0, qat_asym_session_get_private_size(dev));
1212         }
1213 }
1214
1215 static uint16_t
1216 qat_asym_crypto_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
1217                 uint16_t nb_ops)
1218 {
1219         return qat_enqueue_op_burst(qp, qat_asym_build_request, (void **)ops,
1220                         nb_ops);
1221 }
1222
1223 static uint16_t
1224 qat_asym_crypto_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
1225                 uint16_t nb_ops)
1226 {
1227         return qat_dequeue_op_burst(qp, (void **)ops, qat_asym_process_response,
1228                                 nb_ops);
1229 }
1230
1231 void
1232 qat_asym_init_op_cookie(void *op_cookie)
1233 {
1234         int j;
1235         struct qat_asym_op_cookie *cookie = op_cookie;
1236
1237         cookie->input_addr = rte_mempool_virt2iova(cookie) +
1238                         offsetof(struct qat_asym_op_cookie,
1239                                         input_params_ptrs);
1240
1241         cookie->output_addr = rte_mempool_virt2iova(cookie) +
1242                         offsetof(struct qat_asym_op_cookie,
1243                                         output_params_ptrs);
1244
1245         for (j = 0; j < 8; j++) {
1246                 cookie->input_params_ptrs[j] =
1247                                 rte_mempool_virt2iova(cookie) +
1248                                 offsetof(struct qat_asym_op_cookie,
1249                                                 input_array[j]);
1250                 cookie->output_params_ptrs[j] =
1251                                 rte_mempool_virt2iova(cookie) +
1252                                 offsetof(struct qat_asym_op_cookie,
1253                                                 output_array[j]);
1254         }
1255 }
1256
1257 int
1258 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
1259                 struct qat_dev_cmd_param *qat_dev_cmd_param)
1260 {
1261         struct qat_cryptodev_private *internals;
1262         struct rte_cryptodev *cryptodev;
1263         struct qat_device_info *qat_dev_instance =
1264                 &qat_pci_devs[qat_pci_dev->qat_dev_id];
1265         struct rte_cryptodev_pmd_init_params init_params = {
1266                 .name = "",
1267                 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
1268                 .private_data_size = sizeof(struct qat_cryptodev_private)
1269         };
1270         struct qat_capabilities_info capa_info;
1271         const struct rte_cryptodev_capabilities *capabilities;
1272         const struct qat_crypto_gen_dev_ops *gen_dev_ops =
1273                 &qat_asym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
1274         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
1275         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
1276         uint64_t capa_size;
1277         int i = 0;
1278
1279         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
1280                         qat_pci_dev->name, "asym");
1281         QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name);
1282
1283         if (gen_dev_ops->cryptodev_ops == NULL) {
1284                 QAT_LOG(ERR, "Device %s does not support asymmetric crypto",
1285                                 name);
1286                 return -(EFAULT);
1287         }
1288
1289         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
1290                 qat_pci_dev->qat_asym_driver_id =
1291                                 qat_asym_driver_id;
1292         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1293                 if (qat_pci_dev->qat_asym_driver_id !=
1294                                 qat_asym_driver_id) {
1295                         QAT_LOG(ERR,
1296                                 "Device %s have different driver id than corresponding device in primary process",
1297                                 name);
1298                         return -(EFAULT);
1299                 }
1300         }
1301
1302         /* Populate subset device to use in cryptodev device creation */
1303         qat_dev_instance->asym_rte_dev.driver = &cryptodev_qat_asym_driver;
1304         qat_dev_instance->asym_rte_dev.numa_node =
1305                         qat_dev_instance->pci_dev->device.numa_node;
1306         qat_dev_instance->asym_rte_dev.devargs = NULL;
1307
1308         cryptodev = rte_cryptodev_pmd_create(name,
1309                         &(qat_dev_instance->asym_rte_dev), &init_params);
1310
1311         if (cryptodev == NULL)
1312                 return -ENODEV;
1313
1314         qat_dev_instance->asym_rte_dev.name = cryptodev->data->name;
1315         cryptodev->driver_id = qat_asym_driver_id;
1316         cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
1317
1318         cryptodev->enqueue_burst = qat_asym_crypto_enqueue_op_burst;
1319         cryptodev->dequeue_burst = qat_asym_crypto_dequeue_op_burst;
1320
1321         cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
1322
1323         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1324                 return 0;
1325
1326         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
1327                         "QAT_ASYM_CAPA_GEN_%d",
1328                         qat_pci_dev->qat_dev_gen);
1329
1330         internals = cryptodev->data->dev_private;
1331         internals->qat_dev = qat_pci_dev;
1332         internals->dev_id = cryptodev->data->dev_id;
1333
1334         capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
1335         capabilities = capa_info.data;
1336         capa_size = capa_info.size;
1337
1338         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
1339         if (internals->capa_mz == NULL) {
1340                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
1341                                 capa_size, rte_socket_id(), 0);
1342                 if (internals->capa_mz == NULL) {
1343                         QAT_LOG(DEBUG,
1344                                 "Error allocating memzone for capabilities, "
1345                                 "destroying PMD for %s",
1346                                 name);
1347                         rte_cryptodev_pmd_destroy(cryptodev);
1348                         memset(&qat_dev_instance->asym_rte_dev, 0,
1349                                 sizeof(qat_dev_instance->asym_rte_dev));
1350                         return -EFAULT;
1351                 }
1352         }
1353
1354         memcpy(internals->capa_mz->addr, capabilities, capa_size);
1355         internals->qat_dev_capabilities = internals->capa_mz->addr;
1356
1357         while (1) {
1358                 if (qat_dev_cmd_param[i].name == NULL)
1359                         break;
1360                 if (!strcmp(qat_dev_cmd_param[i].name, ASYM_ENQ_THRESHOLD_NAME))
1361                         internals->min_enq_burst_threshold =
1362                                         qat_dev_cmd_param[i].val;
1363                 i++;
1364         }
1365
1366         qat_pci_dev->asym_dev = internals;
1367         internals->service_type = QAT_SERVICE_ASYMMETRIC;
1368         QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
1369                         cryptodev->data->name, internals->dev_id);
1370         return 0;
1371 }
1372
1373 int
1374 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
1375 {
1376         struct rte_cryptodev *cryptodev;
1377
1378         if (qat_pci_dev == NULL)
1379                 return -ENODEV;
1380         if (qat_pci_dev->asym_dev == NULL)
1381                 return 0;
1382         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
1383                 rte_memzone_free(qat_pci_dev->asym_dev->capa_mz);
1384
1385         /* free crypto device */
1386         cryptodev = rte_cryptodev_pmd_get_dev(
1387                         qat_pci_dev->asym_dev->dev_id);
1388         rte_cryptodev_pmd_destroy(cryptodev);
1389         qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
1390         qat_pci_dev->asym_dev = NULL;
1391
1392         return 0;
1393 }
1394
1395 static struct cryptodev_driver qat_crypto_drv;
1396 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
1397                 cryptodev_qat_asym_driver,
1398                 qat_asym_driver_id);