drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include "qat_asym.h"
   8 #include "icp_qat_fw_pke.h"
   9 #include "icp_qat_fw.h"
  10 #include "qat_pke_functionality_arrays.h"
  11
  12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
  13
  14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
  15                 size_t arr_sz, size_t *size, uint32_t *func_id)
  16 {
  17         size_t i;
  18
  19         for (i = 0; i < arr_sz; i++) {
  20                 if (*size <= arr[i][0]) {
  21                         *size = arr[i][0];
  22                         *func_id = arr[i][1];
  23                         return 0;
  24                 }
  25         }
  26         return -1;
  27 }
  28
  29 static inline void qat_fill_req_tmpl(struct icp_qat_fw_pke_request *qat_req)
  30 {
  31         memset(qat_req, 0, sizeof(*qat_req));
  32         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
  33
  34         qat_req->pke_hdr.hdr_flags =
  35                         ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
  36                         (ICP_QAT_FW_COMN_REQ_FLAG_SET);
  37 }
  38
  39 static inline void qat_asym_build_req_tmpl(void *sess_private_data)
  40 {
  41         struct icp_qat_fw_pke_request *qat_req;
  42         struct qat_asym_session *session = sess_private_data;
  43
  44         qat_req = &session->req_tmpl;
  45         qat_fill_req_tmpl(qat_req);
  46 }
  47
  48 static size_t max_of(int n, ...)
  49 {
  50         va_list args;
  51         size_t len = 0, num;
  52         int i;
  53
  54         va_start(args, n);
  55         len = va_arg(args, size_t);
  56
  57         for (i = 0; i < n - 1; i++) {
  58                 num = va_arg(args, size_t);
  59                 if (num > len)
  60                         len = num;
  61         }
  62         va_end(args);
  63
  64         return len;
  65 }
  66
  67 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
  68                 int in_count, int out_count, int alg_size)
  69 {
  70         int i;
  71
  72         for (i = 0; i < in_count; i++)
  73                 memset(cookie->input_array[i], 0x0, alg_size);
  74         for (i = 0; i < out_count; i++)
  75                 memset(cookie->output_array[i], 0x0, alg_size);
  76 }
  77
  78 static void qat_clear_arrays_crt(struct qat_asym_op_cookie *cookie,
  79                 int alg_size)
  80 {
  81         int i;
  82
  83         memset(cookie->input_array[0], 0x0, alg_size);
  84         for (i = 1; i < QAT_ASYM_RSA_QT_NUM_IN_PARAMS; i++)
  85                 memset(cookie->input_array[i], 0x0, alg_size / 2);
  86         for (i = 0; i < QAT_ASYM_RSA_NUM_OUT_PARAMS; i++)
  87                 memset(cookie->output_array[i], 0x0, alg_size);
  88 }
  89
  90 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
  91                 struct rte_crypto_asym_xform *xform, int alg_size)
  92 {
  93         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX)
  94                 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
  95                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, alg_size);
  96         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV)
  97                 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
  98                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, alg_size);
  99         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 100                 if (xform->rsa.key_type == RTE_RSA_KET_TYPE_QT)
 101                         qat_clear_arrays_crt(cookie, alg_size);
 102                 else {
 103                         qat_clear_arrays(cookie, QAT_ASYM_RSA_NUM_IN_PARAMS,
 104                                 QAT_ASYM_RSA_NUM_OUT_PARAMS, alg_size);
 105                 }
 106         }
 107 }
 108
 109 static int qat_asym_check_nonzero(rte_crypto_param n)
 110 {
 111         if (n.length < 8) {
 112                 /* Not a case for any cryptograpic function except for DH
 113                  * generator which very often can be of one byte length
 114                  */
 115                 size_t i;
 116
 117                 if (n.data[n.length - 1] == 0x0) {
 118                         for (i = 0; i < n.length - 1; i++)
 119                                 if (n.data[i] != 0x0)
 120                                         break;
 121                         if (i == n.length - 1)
 122                                 return -(EINVAL);
 123                 }
 124         } else if (*(uint64_t *)&n.data[
 125                                 n.length - 8] == 0) {
 126                 /* Very likely it is zeroed modulus */
 127                 size_t i;
 128
 129                 for (i = 0; i < n.length - 8; i++)
 130                         if (n.data[i] != 0x0)
 131                                 break;
 132                 if (i == n.length - 8)
 133                         return -(EINVAL);
 134         }
 135
 136         return 0;
 137 }
 138
 139 static int
 140 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 141                 struct icp_qat_fw_pke_request *qat_req,
 142                 struct qat_asym_op_cookie *cookie,
 143                 struct rte_crypto_asym_xform *xform)
 144 {
 145         int err = 0;
 146         size_t alg_size;
 147         size_t alg_size_in_bytes;
 148         uint32_t func_id = 0;
 149
 150         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 151                 err = qat_asym_check_nonzero(xform->modex.modulus);
 152                 if (err) {
 153                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
 154                                         " aborting this operation");
 155                         return err;
 156                 }
 157
 158                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
 159                                xform->modex.exponent.length,
 160                                xform->modex.modulus.length);
 161                 alg_size = alg_size_in_bytes << 3;
 162
 163                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
 164                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
 165                                 &alg_size, &func_id)) {
 166                         return -(EINVAL);
 167                 }
 168
 169                 alg_size_in_bytes = alg_size >> 3;
 170                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 171                         asym_op->modex.base.length
 172                         , asym_op->modex.base.data,
 173                         asym_op->modex.base.length);
 174                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 175                         xform->modex.exponent.length
 176                         , xform->modex.exponent.data,
 177                         xform->modex.exponent.length);
 178                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
 179                         xform->modex.modulus.length,
 180                         xform->modex.modulus.data,
 181                         xform->modex.modulus.length);
 182                 cookie->alg_size = alg_size;
 183                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 184                 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
 185                 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
 186 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 187                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
 188                                 cookie->input_array[0],
 189                                 alg_size_in_bytes);
 190                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
 191                                 cookie->input_array[1],
 192                                 alg_size_in_bytes);
 193                 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
 194                                 cookie->input_array[2],
 195                                 alg_size_in_bytes);
 196 #endif
 197         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 198                 err = qat_asym_check_nonzero(xform->modinv.modulus);
 199                 if (err) {
 200                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
 201                                         " inverse, aborting this operation");
 202                         return err;
 203                 }
 204
 205                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
 206                                 xform->modinv.modulus.length);
 207                 alg_size = alg_size_in_bytes << 3;
 208
 209                 if (xform->modinv.modulus.data[
 210                                 xform->modinv.modulus.length - 1] & 0x01) {
 211                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
 212                                         sizeof(MOD_INV_IDS_ODD)/
 213                                         sizeof(*MOD_INV_IDS_ODD),
 214                                         &alg_size, &func_id)) {
 215                                 return -(EINVAL);
 216                         }
 217                 } else {
 218                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
 219                                         sizeof(MOD_INV_IDS_EVEN)/
 220                                         sizeof(*MOD_INV_IDS_EVEN),
 221                                         &alg_size, &func_id)) {
 222                                 return -(EINVAL);
 223                         }
 224                 }
 225
 226                 alg_size_in_bytes = alg_size >> 3;
 227                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 228                         asym_op->modinv.base.length
 229                                 , asym_op->modinv.base.data,
 230                                 asym_op->modinv.base.length);
 231                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 232                                 xform->modinv.modulus.length
 233                                 , xform->modinv.modulus.data,
 234                                 xform->modinv.modulus.length);
 235                 cookie->alg_size = alg_size;
 236                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 237                 qat_req->input_param_count =
 238                                 QAT_ASYM_MODINV_NUM_IN_PARAMS;
 239                 qat_req->output_param_count =
 240                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
 241 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 242                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
 243                                 cookie->input_array[0],
 244                                 alg_size_in_bytes);
 245                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
 246                                 cookie->input_array[1],
 247                                 alg_size_in_bytes);
 248 #endif
 249         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 250                 err = qat_asym_check_nonzero(xform->rsa.n);
 251                 if (err) {
 252                         QAT_LOG(ERR, "Empty modulus in RSA"
 253                                         " inverse, aborting this operation");
 254                         return err;
 255                 }
 256
 257                 alg_size_in_bytes = xform->rsa.n.length;
 258                 alg_size = alg_size_in_bytes << 3;
 259
 260                 qat_req->input_param_count =
 261                                 QAT_ASYM_RSA_NUM_IN_PARAMS;
 262                 qat_req->output_param_count =
 263                                 QAT_ASYM_RSA_NUM_OUT_PARAMS;
 264
 265                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 266                                 asym_op->rsa.op_type ==
 267                                                 RTE_CRYPTO_ASYM_OP_VERIFY) {
 268
 269                         if (qat_asym_get_sz_and_func_id(RSA_ENC_IDS,
 270                                         sizeof(RSA_ENC_IDS)/
 271                                         sizeof(*RSA_ENC_IDS),
 272                                         &alg_size, &func_id)) {
 273                                 err = -(EINVAL);
 274                                 QAT_LOG(ERR,
 275                                         "Not supported RSA parameter size (key)");
 276                                 return err;
 277                         }
 278                         alg_size_in_bytes = alg_size >> 3;
 279                         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 280                                 switch (asym_op->rsa.pad) {
 281                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 282                                         rte_memcpy(cookie->input_array[0] +
 283                                                 alg_size_in_bytes -
 284                                                 asym_op->rsa.message.length
 285                                                 , asym_op->rsa.message.data,
 286                                                 asym_op->rsa.message.length);
 287                                         break;
 288                                 default:
 289                                         err = -(EINVAL);
 290                                         QAT_LOG(ERR,
 291                                                 "Invalid RSA padding (Encryption)");
 292                                         return err;
 293                                 }
 294 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 295                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Message",
 296                                                 cookie->input_array[0],
 297                                                 alg_size_in_bytes);
 298 #endif
 299                         } else {
 300                                 switch (asym_op->rsa.pad) {
 301                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 302                                         rte_memcpy(cookie->input_array[0],
 303                                                 asym_op->rsa.sign.data,
 304                                                 alg_size_in_bytes);
 305                                         break;
 306                                 default:
 307                                         err = -(EINVAL);
 308                                         QAT_LOG(ERR,
 309                                                 "Invalid RSA padding (Verify)");
 310                                         return err;
 311                                 }
 312
 313 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 314                                 QAT_DP_HEXDUMP_LOG(DEBUG, " RSA Signature",
 315                                                 cookie->input_array[0],
 316                                                 alg_size_in_bytes);
 317 #endif
 318
 319                         }
 320                         rte_memcpy(cookie->input_array[1] +
 321                                         alg_size_in_bytes -
 322                                         xform->rsa.e.length
 323                                         , xform->rsa.e.data,
 324                                         xform->rsa.e.length);
 325                         rte_memcpy(cookie->input_array[2] +
 326                                         alg_size_in_bytes -
 327                                         xform->rsa.n.length,
 328                                         xform->rsa.n.data,
 329                                         xform->rsa.n.length);
 330
 331                         cookie->alg_size = alg_size;
 332                         qat_req->pke_hdr.cd_pars.func_id = func_id;
 333
 334 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 335                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Public Key",
 336                                         cookie->input_array[1], alg_size_in_bytes);
 337                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Modulus",
 338                                         cookie->input_array[2], alg_size_in_bytes);
 339 #endif
 340                 } else {
 341                         if (asym_op->rsa.op_type ==
 342                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 343                                 switch (asym_op->rsa.pad) {
 344                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 345                                         rte_memcpy(cookie->input_array[0]
 346                                                 + alg_size_in_bytes -
 347                                                 asym_op->rsa.cipher.length,
 348                                                 asym_op->rsa.cipher.data,
 349                                                 asym_op->rsa.cipher.length);
 350                                         break;
 351                                 default:
 352                                         QAT_LOG(ERR,
 353                                                 "Invalid padding of RSA (Decrypt)");
 354                                         return -(EINVAL);
 355                                 }
 356
 357                         } else if (asym_op->rsa.op_type ==
 358                                         RTE_CRYPTO_ASYM_OP_SIGN) {
 359                                 switch (asym_op->rsa.pad) {
 360                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 361                                         rte_memcpy(cookie->input_array[0]
 362                                                 + alg_size_in_bytes -
 363                                                 asym_op->rsa.message.length,
 364                                                 asym_op->rsa.message.data,
 365                                                 asym_op->rsa.message.length);
 366                                         break;
 367                                 default:
 368                                         QAT_LOG(ERR,
 369                                                 "Invalid padding of RSA (Signature)");
 370                                         return -(EINVAL);
 371                                 }
 372                         }
 373                         if (xform->rsa.key_type == RTE_RSA_KET_TYPE_QT) {
 374
 375                                 qat_req->input_param_count =
 376                                                 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
 377                                 if (qat_asym_get_sz_and_func_id(RSA_DEC_CRT_IDS,
 378                                                 sizeof(RSA_DEC_CRT_IDS)/
 379                                                 sizeof(*RSA_DEC_CRT_IDS),
 380                                                 &alg_size, &func_id)) {
 381                                         return -(EINVAL);
 382                                 }
 383                                 alg_size_in_bytes = alg_size >> 3;
 384
 385                                 rte_memcpy(cookie->input_array[1] +
 386                                                 (alg_size_in_bytes >> 1) -
 387                                                 xform->rsa.qt.p.length
 388                                                 , xform->rsa.qt.p.data,
 389                                                 xform->rsa.qt.p.length);
 390                                 rte_memcpy(cookie->input_array[2] +
 391                                                 (alg_size_in_bytes >> 1) -
 392                                                 xform->rsa.qt.q.length
 393                                                 , xform->rsa.qt.q.data,
 394                                                 xform->rsa.qt.q.length);
 395                                 rte_memcpy(cookie->input_array[3] +
 396                                                 (alg_size_in_bytes >> 1) -
 397                                                 xform->rsa.qt.dP.length
 398                                                 , xform->rsa.qt.dP.data,
 399                                                 xform->rsa.qt.dP.length);
 400                                 rte_memcpy(cookie->input_array[4] +
 401                                                 (alg_size_in_bytes >> 1) -
 402                                                 xform->rsa.qt.dQ.length
 403                                                 , xform->rsa.qt.dQ.data,
 404                                                 xform->rsa.qt.dQ.length);
 405                                 rte_memcpy(cookie->input_array[5] +
 406                                                 (alg_size_in_bytes >> 1) -
 407                                                 xform->rsa.qt.qInv.length
 408                                                 , xform->rsa.qt.qInv.data,
 409                                                 xform->rsa.qt.qInv.length);
 410                                 cookie->alg_size = alg_size;
 411                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 412
 413 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 414                                 QAT_DP_HEXDUMP_LOG(DEBUG, "C",
 415                                                 cookie->input_array[0],
 416                                                 alg_size_in_bytes);
 417                                 QAT_DP_HEXDUMP_LOG(DEBUG, "p",
 418                                                 cookie->input_array[1],
 419                                                 alg_size_in_bytes);
 420                                 QAT_DP_HEXDUMP_LOG(DEBUG, "q",
 421                                                 cookie->input_array[2],
 422                                                 alg_size_in_bytes);
 423                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 424                                                 "dP", cookie->input_array[3],
 425                                                 alg_size_in_bytes);
 426                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 427                                                 "dQ", cookie->input_array[4],
 428                                                 alg_size_in_bytes);
 429                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 430                                                 "qInv", cookie->input_array[5],
 431                                                 alg_size_in_bytes);
 432 #endif
 433                         } else if (xform->rsa.key_type ==
 434                                         RTE_RSA_KEY_TYPE_EXP) {
 435                                 if (qat_asym_get_sz_and_func_id(
 436                                                 RSA_DEC_IDS,
 437                                                 sizeof(RSA_DEC_IDS)/
 438                                                 sizeof(*RSA_DEC_IDS),
 439                                                 &alg_size, &func_id)) {
 440                                         return -(EINVAL);
 441                                 }
 442                                 alg_size_in_bytes = alg_size >> 3;
 443                                 rte_memcpy(cookie->input_array[1] +
 444                                                 alg_size_in_bytes -
 445                                                 xform->rsa.d.length,
 446                                                 xform->rsa.d.data,
 447                                                 xform->rsa.d.length);
 448                                 rte_memcpy(cookie->input_array[2] +
 449                                                 alg_size_in_bytes -
 450                                                 xform->rsa.n.length,
 451                                                 xform->rsa.n.data,
 452                                                 xform->rsa.n.length);
 453 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 454                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA ciphertext",
 455                                         cookie->input_array[0],
 456                                         alg_size_in_bytes);
 457                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA d", cookie->input_array[1],
 458                                         alg_size_in_bytes);
 459                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA n", cookie->input_array[2],
 460                                         alg_size_in_bytes);
 461 #endif
 462
 463                                 cookie->alg_size = alg_size;
 464                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 465                         } else {
 466                                 QAT_LOG(ERR, "Invalid RSA key type");
 467                                 return -(EINVAL);
 468                         }
 469                 }
 470         } else {
 471                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 472                 return -(EINVAL);
 473         }
 474         return 0;
 475 }
 476
 477 int
 478 qat_asym_build_request(void *in_op,
 479                         uint8_t *out_msg,
 480                         void *op_cookie,
 481                         __rte_unused enum qat_device_gen qat_dev_gen)
 482 {
 483         struct qat_asym_session *ctx;
 484         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 485         struct rte_crypto_asym_op *asym_op = op->asym;
 486         struct icp_qat_fw_pke_request *qat_req =
 487                         (struct icp_qat_fw_pke_request *)out_msg;
 488         struct qat_asym_op_cookie *cookie =
 489                                 (struct qat_asym_op_cookie *)op_cookie;
 490         int err = 0;
 491
 492         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 493         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 494                 ctx = (struct qat_asym_session *)
 495                         get_asym_session_private_data(
 496                         op->asym->session, qat_asym_driver_id);
 497                 if (unlikely(ctx == NULL)) {
 498                         QAT_LOG(ERR, "Session has not been created for this device");
 499                         goto error;
 500                 }
 501                 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
 502                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
 503                 if (err) {
 504                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 505                         goto error;
 506                 }
 507         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 508                 qat_fill_req_tmpl(qat_req);
 509                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
 510                                 op->asym->xform);
 511                 if (err) {
 512                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 513                         goto error;
 514                 }
 515         } else {
 516                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
 517                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 518                 goto error;
 519         }
 520
 521         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 522         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 523         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 524
 525 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 526         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 527                         sizeof(struct icp_qat_fw_pke_request));
 528 #endif
 529
 530         return 0;
 531 error:
 532
 533         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 534
 535 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 536         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 537                 sizeof(struct icp_qat_fw_pke_request));
 538 #endif
 539
 540         qat_req->output_param_count = 0;
 541         qat_req->input_param_count = 0;
 542         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 543         cookie->error |= err;
 544
 545         return 0;
 546 }
 547
 548 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
 549                 struct qat_asym_op_cookie *cookie,
 550                 struct rte_crypto_asym_xform *xform)
 551 {
 552         size_t alg_size, alg_size_in_bytes = 0;
 553         struct rte_crypto_asym_op *asym_op = rx_op->asym;
 554
 555         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 556                 rte_crypto_param n = xform->modex.modulus;
 557
 558                 alg_size = cookie->alg_size;
 559                 alg_size_in_bytes = alg_size >> 3;
 560                 uint8_t *modexp_result = asym_op->modex.result.data;
 561
 562                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 563                         rte_memcpy(modexp_result +
 564                                 (asym_op->modex.result.length -
 565                                         n.length),
 566                                 cookie->output_array[0] + alg_size_in_bytes
 567                                 - n.length, n.length
 568                                 );
 569                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 570 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 571                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
 572                                         cookie->output_array[0],
 573                                         alg_size_in_bytes);
 574
 575 #endif
 576                 }
 577         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 578                 rte_crypto_param n = xform->modinv.modulus;
 579
 580                 alg_size = cookie->alg_size;
 581                 alg_size_in_bytes = alg_size >> 3;
 582                 uint8_t *modinv_result = asym_op->modinv.result.data;
 583
 584                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 585                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 586                                 - n.length),
 587                                 cookie->output_array[0] + alg_size_in_bytes
 588                                 - n.length, n.length);
 589                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 590 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 591                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
 592                                         cookie->output_array[0],
 593                                         alg_size_in_bytes);
 594 #endif
 595                 }
 596         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 597
 598                 alg_size = cookie->alg_size;
 599                 alg_size_in_bytes = alg_size >> 3;
 600                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 601                                 asym_op->rsa.op_type ==
 602                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 603                         if (asym_op->rsa.op_type ==
 604                                         RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 605                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 606
 607                                 rte_memcpy(rsa_result,
 608                                                 cookie->output_array[0],
 609                                                 alg_size_in_bytes);
 610                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 611 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 612                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Encrypted data",
 613                                                 cookie->output_array[0],
 614                                                 alg_size_in_bytes);
 615 #endif
 616                         } else if (asym_op->rsa.op_type ==
 617                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 618                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 619
 620                                 switch (asym_op->rsa.pad) {
 621                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 622                                         rte_memcpy(rsa_result,
 623                                                         cookie->output_array[0],
 624                                                         alg_size_in_bytes);
 625                                         rx_op->status =
 626                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 627 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 628                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 629                                                 cookie->output_array[0],
 630                                                 alg_size_in_bytes);
 631 #endif
 632                                         break;
 633                                 default:
 634                                         QAT_LOG(ERR, "Padding not supported");
 635                                         rx_op->status =
 636                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 637                                         break;
 638                                 }
 639                         }
 640                 } else {
 641                         if (asym_op->rsa.op_type ==
 642                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 643                                 uint8_t *rsa_result = asym_op->rsa.message.data;
 644
 645                                 switch (asym_op->rsa.pad) {
 646                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 647                                         rte_memcpy(rsa_result,
 648                                                 cookie->output_array[0],
 649                                                 alg_size_in_bytes);
 650                                         rx_op->status =
 651                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 652                                         break;
 653                                 default:
 654                                         QAT_LOG(ERR, "Padding not supported");
 655                                         rx_op->status =
 656                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 657                                         break;
 658                                 }
 659 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 660                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Decrypted Message",
 661                                                 rsa_result, alg_size_in_bytes);
 662 #endif
 663                         } else if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
 664                                 uint8_t *rsa_result = asym_op->rsa.sign.data;
 665
 666                                 rte_memcpy(rsa_result,
 667                                                 cookie->output_array[0],
 668                                                 alg_size_in_bytes);
 669                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 670 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 671                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 672                                                 cookie->output_array[0],
 673                                                 alg_size_in_bytes);
 674 #endif
 675                         }
 676                 }
 677         }
 678         qat_clear_arrays_by_alg(cookie, xform, alg_size_in_bytes);
 679 }
 680
 681 void
 682 qat_asym_process_response(void **op, uint8_t *resp,
 683                 void *op_cookie)
 684 {
 685         struct qat_asym_session *ctx;
 686         struct icp_qat_fw_pke_resp *resp_msg =
 687                         (struct icp_qat_fw_pke_resp *)resp;
 688         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
 689                         (resp_msg->opaque);
 690         struct qat_asym_op_cookie *cookie = op_cookie;
 691
 692         if (cookie->error) {
 693                 cookie->error = 0;
 694                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 695                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 696                 QAT_DP_LOG(ERR, "Cookie status returned error");
 697         } else {
 698                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 699                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 700                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 701                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 702                         QAT_DP_LOG(ERR, "Asymmetric response status"
 703                                         " returned error");
 704                 }
 705                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 706                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 707                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 708                         QAT_DP_LOG(ERR, "Asymmetric common status"
 709                                         " returned error");
 710                 }
 711         }
 712
 713         if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 714                 ctx = (struct qat_asym_session *)get_asym_session_private_data(
 715                         rx_op->asym->session, qat_asym_driver_id);
 716                 qat_asym_collect_response(rx_op, cookie, ctx->xform);
 717         } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 718                 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
 719         }
 720         *op = rx_op;
 721
 722 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 723         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
 724                         sizeof(struct icp_qat_fw_pke_resp));
 725 #endif
 726 }
 727
 728 int
 729 qat_asym_session_configure(struct rte_cryptodev *dev,
 730                 struct rte_crypto_asym_xform *xform,
 731                 struct rte_cryptodev_asym_session *sess,
 732                 struct rte_mempool *mempool)
 733 {
 734         int err = 0;
 735         void *sess_private_data;
 736         struct qat_asym_session *session;
 737
 738         if (rte_mempool_get(mempool, &sess_private_data)) {
 739                 QAT_LOG(ERR,
 740                         "Couldn't get object from session mempool");
 741                 return -ENOMEM;
 742         }
 743
 744         session = sess_private_data;
 745         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 746                 if (xform->modex.exponent.length == 0 ||
 747                                 xform->modex.modulus.length == 0) {
 748                         QAT_LOG(ERR, "Invalid mod exp input parameter");
 749                         err = -EINVAL;
 750                         goto error;
 751                 }
 752         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 753                 if (xform->modinv.modulus.length == 0) {
 754                         QAT_LOG(ERR, "Invalid mod inv input parameter");
 755                         err = -EINVAL;
 756                         goto error;
 757                 }
 758         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 759                 if (xform->rsa.n.length == 0) {
 760                         QAT_LOG(ERR, "Invalid rsa input parameter");
 761                         err = -EINVAL;
 762                         goto error;
 763                 }
 764         } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
 765                         || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
 766                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 767                 err = -EINVAL;
 768                 goto error;
 769         } else {
 770                 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
 771                 err = -EINVAL;
 772                 goto error;
 773         }
 774
 775         session->xform = xform;
 776         qat_asym_build_req_tmpl(sess_private_data);
 777         set_asym_session_private_data(sess, dev->driver_id,
 778                 sess_private_data);
 779
 780         return 0;
 781 error:
 782         rte_mempool_put(mempool, sess_private_data);
 783         return err;
 784 }
 785
 786 unsigned int qat_asym_session_get_private_size(
 787                 struct rte_cryptodev *dev __rte_unused)
 788 {
 789         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
 790 }
 791
 792 void
 793 qat_asym_session_clear(struct rte_cryptodev *dev,
 794                 struct rte_cryptodev_asym_session *sess)
 795 {
 796         uint8_t index = dev->driver_id;
 797         void *sess_priv = get_asym_session_private_data(sess, index);
 798         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
 799
 800         if (sess_priv) {
 801                 memset(s, 0, qat_asym_session_get_private_size(dev));
 802                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 803
 804                 set_asym_session_private_data(sess, index, NULL);
 805                 rte_mempool_put(sess_mp, sess_priv);
 806         }
 807 }