drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include "qat_asym.h"
   8 #include "icp_qat_fw_pke.h"
   9 #include "icp_qat_fw.h"
  10 #include "qat_pke_functionality_arrays.h"
  11
  12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
  13
  14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
  15                 size_t arr_sz, size_t *size, uint32_t *func_id)
  16 {
  17         size_t i;
  18
  19         for (i = 0; i < arr_sz; i++) {
  20                 if (*size <= arr[i][0]) {
  21                         *size = arr[i][0];
  22                         *func_id = arr[i][1];
  23                         return 0;
  24                 }
  25         }
  26         return -1;
  27 }
  28
  29 static inline void qat_fill_req_tmpl(struct icp_qat_fw_pke_request *qat_req)
  30 {
  31         memset(qat_req, 0, sizeof(*qat_req));
  32         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
  33
  34         qat_req->pke_hdr.hdr_flags =
  35                         ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
  36                         (ICP_QAT_FW_COMN_REQ_FLAG_SET);
  37 }
  38
  39 static inline void qat_asym_build_req_tmpl(void *sess_private_data)
  40 {
  41         struct icp_qat_fw_pke_request *qat_req;
  42         struct qat_asym_session *session = sess_private_data;
  43
  44         qat_req = &session->req_tmpl;
  45         qat_fill_req_tmpl(qat_req);
  46 }
  47
  48 static size_t max_of(int n, ...)
  49 {
  50         va_list args;
  51         size_t len = 0, num;
  52         int i;
  53
  54         va_start(args, n);
  55         len = va_arg(args, size_t);
  56
  57         for (i = 0; i < n - 1; i++) {
  58                 num = va_arg(args, size_t);
  59                 if (num > len)
  60                         len = num;
  61         }
  62         va_end(args);
  63
  64         return len;
  65 }
  66
  67 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
  68                 int in_count, int out_count, int alg_size)
  69 {
  70         int i;
  71
  72         for (i = 0; i < in_count; i++)
  73                 memset(cookie->input_array[i], 0x0, alg_size);
  74         for (i = 0; i < out_count; i++)
  75                 memset(cookie->output_array[i], 0x0, alg_size);
  76 }
  77
  78 static void qat_clear_arrays_crt(struct qat_asym_op_cookie *cookie,
  79                 int alg_size)
  80 {
  81         int i;
  82
  83         memset(cookie->input_array[0], 0x0, alg_size);
  84         for (i = 1; i < QAT_ASYM_RSA_QT_NUM_IN_PARAMS; i++)
  85                 memset(cookie->input_array[i], 0x0, alg_size / 2);
  86         for (i = 0; i < QAT_ASYM_RSA_NUM_OUT_PARAMS; i++)
  87                 memset(cookie->output_array[i], 0x0, alg_size);
  88 }
  89
  90 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
  91                 struct rte_crypto_asym_xform *xform, int alg_size)
  92 {
  93         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX)
  94                 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
  95                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, alg_size);
  96         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV)
  97                 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
  98                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, alg_size);
  99         else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 100                 if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT)
 101                         qat_clear_arrays_crt(cookie, alg_size);
 102                 else {
 103                         qat_clear_arrays(cookie, QAT_ASYM_RSA_NUM_IN_PARAMS,
 104                                 QAT_ASYM_RSA_NUM_OUT_PARAMS, alg_size);
 105                 }
 106         }
 107 }
 108
 109 static int qat_asym_check_nonzero(rte_crypto_param n)
 110 {
 111         if (n.length < 8) {
 112                 /* Not a case for any cryptographic function except for DH
 113                  * generator which very often can be of one byte length
 114                  */
 115                 size_t i;
 116
 117                 if (n.data[n.length - 1] == 0x0) {
 118                         for (i = 0; i < n.length - 1; i++)
 119                                 if (n.data[i] != 0x0)
 120                                         break;
 121                         if (i == n.length - 1)
 122                                 return -(EINVAL);
 123                 }
 124         } else if (*(uint64_t *)&n.data[
 125                                 n.length - 8] == 0) {
 126                 /* Very likely it is zeroed modulus */
 127                 size_t i;
 128
 129                 for (i = 0; i < n.length - 8; i++)
 130                         if (n.data[i] != 0x0)
 131                                 break;
 132                 if (i == n.length - 8)
 133                         return -(EINVAL);
 134         }
 135
 136         return 0;
 137 }
 138
 139 static int
 140 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 141                 struct icp_qat_fw_pke_request *qat_req,
 142                 struct qat_asym_op_cookie *cookie,
 143                 struct rte_crypto_asym_xform *xform)
 144 {
 145         int err = 0;
 146         size_t alg_size;
 147         size_t alg_size_in_bytes;
 148         uint32_t func_id = 0;
 149
 150         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 151                 err = qat_asym_check_nonzero(xform->modex.modulus);
 152                 if (err) {
 153                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
 154                                         " aborting this operation");
 155                         return err;
 156                 }
 157
 158                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
 159                                xform->modex.exponent.length,
 160                                xform->modex.modulus.length);
 161                 alg_size = alg_size_in_bytes << 3;
 162
 163                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
 164                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
 165                                 &alg_size, &func_id)) {
 166                         return -(EINVAL);
 167                 }
 168
 169                 alg_size_in_bytes = alg_size >> 3;
 170                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 171                         asym_op->modex.base.length
 172                         , asym_op->modex.base.data,
 173                         asym_op->modex.base.length);
 174                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 175                         xform->modex.exponent.length
 176                         , xform->modex.exponent.data,
 177                         xform->modex.exponent.length);
 178                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
 179                         xform->modex.modulus.length,
 180                         xform->modex.modulus.data,
 181                         xform->modex.modulus.length);
 182                 cookie->alg_size = alg_size;
 183                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 184                 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
 185                 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
 186 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 187                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
 188                                 cookie->input_array[0],
 189                                 alg_size_in_bytes);
 190                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
 191                                 cookie->input_array[1],
 192                                 alg_size_in_bytes);
 193                 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
 194                                 cookie->input_array[2],
 195                                 alg_size_in_bytes);
 196 #endif
 197         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 198                 err = qat_asym_check_nonzero(xform->modinv.modulus);
 199                 if (err) {
 200                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
 201                                         " inverse, aborting this operation");
 202                         return err;
 203                 }
 204
 205                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
 206                                 xform->modinv.modulus.length);
 207                 alg_size = alg_size_in_bytes << 3;
 208
 209                 if (xform->modinv.modulus.data[
 210                                 xform->modinv.modulus.length - 1] & 0x01) {
 211                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
 212                                         sizeof(MOD_INV_IDS_ODD)/
 213                                         sizeof(*MOD_INV_IDS_ODD),
 214                                         &alg_size, &func_id)) {
 215                                 return -(EINVAL);
 216                         }
 217                 } else {
 218                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
 219                                         sizeof(MOD_INV_IDS_EVEN)/
 220                                         sizeof(*MOD_INV_IDS_EVEN),
 221                                         &alg_size, &func_id)) {
 222                                 return -(EINVAL);
 223                         }
 224                 }
 225
 226                 alg_size_in_bytes = alg_size >> 3;
 227                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 228                         asym_op->modinv.base.length
 229                                 , asym_op->modinv.base.data,
 230                                 asym_op->modinv.base.length);
 231                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 232                                 xform->modinv.modulus.length
 233                                 , xform->modinv.modulus.data,
 234                                 xform->modinv.modulus.length);
 235                 cookie->alg_size = alg_size;
 236                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 237                 qat_req->input_param_count =
 238                                 QAT_ASYM_MODINV_NUM_IN_PARAMS;
 239                 qat_req->output_param_count =
 240                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
 241 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 242                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
 243                                 cookie->input_array[0],
 244                                 alg_size_in_bytes);
 245                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
 246                                 cookie->input_array[1],
 247                                 alg_size_in_bytes);
 248 #endif
 249         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 250                 err = qat_asym_check_nonzero(xform->rsa.n);
 251                 if (err) {
 252                         QAT_LOG(ERR, "Empty modulus in RSA"
 253                                         " inverse, aborting this operation");
 254                         return err;
 255                 }
 256
 257                 alg_size_in_bytes = xform->rsa.n.length;
 258                 alg_size = alg_size_in_bytes << 3;
 259
 260                 qat_req->input_param_count =
 261                                 QAT_ASYM_RSA_NUM_IN_PARAMS;
 262                 qat_req->output_param_count =
 263                                 QAT_ASYM_RSA_NUM_OUT_PARAMS;
 264
 265                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 266                                 asym_op->rsa.op_type ==
 267                                                 RTE_CRYPTO_ASYM_OP_VERIFY) {
 268
 269                         if (qat_asym_get_sz_and_func_id(RSA_ENC_IDS,
 270                                         sizeof(RSA_ENC_IDS)/
 271                                         sizeof(*RSA_ENC_IDS),
 272                                         &alg_size, &func_id)) {
 273                                 err = -(EINVAL);
 274                                 QAT_LOG(ERR,
 275                                         "Not supported RSA parameter size (key)");
 276                                 return err;
 277                         }
 278                         alg_size_in_bytes = alg_size >> 3;
 279                         if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 280                                 switch (asym_op->rsa.pad) {
 281                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 282                                         rte_memcpy(cookie->input_array[0] +
 283                                                 alg_size_in_bytes -
 284                                                 asym_op->rsa.message.length
 285                                                 , asym_op->rsa.message.data,
 286                                                 asym_op->rsa.message.length);
 287                                         break;
 288                                 default:
 289                                         err = -(EINVAL);
 290                                         QAT_LOG(ERR,
 291                                                 "Invalid RSA padding (Encryption)");
 292                                         return err;
 293                                 }
 294 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 295                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Message",
 296                                                 cookie->input_array[0],
 297                                                 alg_size_in_bytes);
 298 #endif
 299                         } else {
 300                                 switch (asym_op->rsa.pad) {
 301                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 302                                         rte_memcpy(cookie->input_array[0],
 303                                                 asym_op->rsa.sign.data,
 304                                                 alg_size_in_bytes);
 305                                         break;
 306                                 default:
 307                                         err = -(EINVAL);
 308                                         QAT_LOG(ERR,
 309                                                 "Invalid RSA padding (Verify)");
 310                                         return err;
 311                                 }
 312
 313 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 314                                 QAT_DP_HEXDUMP_LOG(DEBUG, " RSA Signature",
 315                                                 cookie->input_array[0],
 316                                                 alg_size_in_bytes);
 317 #endif
 318
 319                         }
 320                         rte_memcpy(cookie->input_array[1] +
 321                                         alg_size_in_bytes -
 322                                         xform->rsa.e.length
 323                                         , xform->rsa.e.data,
 324                                         xform->rsa.e.length);
 325                         rte_memcpy(cookie->input_array[2] +
 326                                         alg_size_in_bytes -
 327                                         xform->rsa.n.length,
 328                                         xform->rsa.n.data,
 329                                         xform->rsa.n.length);
 330
 331                         cookie->alg_size = alg_size;
 332                         qat_req->pke_hdr.cd_pars.func_id = func_id;
 333
 334 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 335                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Public Key",
 336                                         cookie->input_array[1], alg_size_in_bytes);
 337                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Modulus",
 338                                         cookie->input_array[2], alg_size_in_bytes);
 339 #endif
 340                 } else {
 341                         if (asym_op->rsa.op_type ==
 342                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 343                                 switch (asym_op->rsa.pad) {
 344                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 345                                         rte_memcpy(cookie->input_array[0]
 346                                                 + alg_size_in_bytes -
 347                                                 asym_op->rsa.cipher.length,
 348                                                 asym_op->rsa.cipher.data,
 349                                                 asym_op->rsa.cipher.length);
 350                                         break;
 351                                 default:
 352                                         QAT_LOG(ERR,
 353                                                 "Invalid padding of RSA (Decrypt)");
 354                                         return -(EINVAL);
 355                                 }
 356
 357                         } else if (asym_op->rsa.op_type ==
 358                                         RTE_CRYPTO_ASYM_OP_SIGN) {
 359                                 switch (asym_op->rsa.pad) {
 360                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 361                                         rte_memcpy(cookie->input_array[0]
 362                                                 + alg_size_in_bytes -
 363                                                 asym_op->rsa.message.length,
 364                                                 asym_op->rsa.message.data,
 365                                                 asym_op->rsa.message.length);
 366                                         break;
 367                                 default:
 368                                         QAT_LOG(ERR,
 369                                                 "Invalid padding of RSA (Signature)");
 370                                         return -(EINVAL);
 371                                 }
 372                         }
 373                         if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_QT) {
 374
 375                                 qat_req->input_param_count =
 376                                                 QAT_ASYM_RSA_QT_NUM_IN_PARAMS;
 377                                 if (qat_asym_get_sz_and_func_id(RSA_DEC_CRT_IDS,
 378                                                 sizeof(RSA_DEC_CRT_IDS)/
 379                                                 sizeof(*RSA_DEC_CRT_IDS),
 380                                                 &alg_size, &func_id)) {
 381                                         return -(EINVAL);
 382                                 }
 383                                 alg_size_in_bytes = alg_size >> 3;
 384
 385                                 rte_memcpy(cookie->input_array[1] +
 386                                                 (alg_size_in_bytes >> 1) -
 387                                                 xform->rsa.qt.p.length
 388                                                 , xform->rsa.qt.p.data,
 389                                                 xform->rsa.qt.p.length);
 390                                 rte_memcpy(cookie->input_array[2] +
 391                                                 (alg_size_in_bytes >> 1) -
 392                                                 xform->rsa.qt.q.length
 393                                                 , xform->rsa.qt.q.data,
 394                                                 xform->rsa.qt.q.length);
 395                                 rte_memcpy(cookie->input_array[3] +
 396                                                 (alg_size_in_bytes >> 1) -
 397                                                 xform->rsa.qt.dP.length
 398                                                 , xform->rsa.qt.dP.data,
 399                                                 xform->rsa.qt.dP.length);
 400                                 rte_memcpy(cookie->input_array[4] +
 401                                                 (alg_size_in_bytes >> 1) -
 402                                                 xform->rsa.qt.dQ.length
 403                                                 , xform->rsa.qt.dQ.data,
 404                                                 xform->rsa.qt.dQ.length);
 405                                 rte_memcpy(cookie->input_array[5] +
 406                                                 (alg_size_in_bytes >> 1) -
 407                                                 xform->rsa.qt.qInv.length
 408                                                 , xform->rsa.qt.qInv.data,
 409                                                 xform->rsa.qt.qInv.length);
 410                                 cookie->alg_size = alg_size;
 411                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 412
 413 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 414                                 QAT_DP_HEXDUMP_LOG(DEBUG, "C",
 415                                                 cookie->input_array[0],
 416                                                 alg_size_in_bytes);
 417                                 QAT_DP_HEXDUMP_LOG(DEBUG, "p",
 418                                                 cookie->input_array[1],
 419                                                 alg_size_in_bytes);
 420                                 QAT_DP_HEXDUMP_LOG(DEBUG, "q",
 421                                                 cookie->input_array[2],
 422                                                 alg_size_in_bytes);
 423                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 424                                                 "dP", cookie->input_array[3],
 425                                                 alg_size_in_bytes);
 426                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 427                                                 "dQ", cookie->input_array[4],
 428                                                 alg_size_in_bytes);
 429                                 QAT_DP_HEXDUMP_LOG(DEBUG,
 430                                                 "qInv", cookie->input_array[5],
 431                                                 alg_size_in_bytes);
 432 #endif
 433                         } else if (xform->rsa.key_type ==
 434                                         RTE_RSA_KEY_TYPE_EXP) {
 435                                 if (qat_asym_get_sz_and_func_id(
 436                                                 RSA_DEC_IDS,
 437                                                 sizeof(RSA_DEC_IDS)/
 438                                                 sizeof(*RSA_DEC_IDS),
 439                                                 &alg_size, &func_id)) {
 440                                         return -(EINVAL);
 441                                 }
 442                                 alg_size_in_bytes = alg_size >> 3;
 443                                 rte_memcpy(cookie->input_array[1] +
 444                                                 alg_size_in_bytes -
 445                                                 xform->rsa.d.length,
 446                                                 xform->rsa.d.data,
 447                                                 xform->rsa.d.length);
 448                                 rte_memcpy(cookie->input_array[2] +
 449                                                 alg_size_in_bytes -
 450                                                 xform->rsa.n.length,
 451                                                 xform->rsa.n.data,
 452                                                 xform->rsa.n.length);
 453 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 454                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA ciphertext",
 455                                         cookie->input_array[0],
 456                                         alg_size_in_bytes);
 457                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA d", cookie->input_array[1],
 458                                         alg_size_in_bytes);
 459                         QAT_DP_HEXDUMP_LOG(DEBUG, "RSA n", cookie->input_array[2],
 460                                         alg_size_in_bytes);
 461 #endif
 462
 463                                 cookie->alg_size = alg_size;
 464                                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 465                         } else {
 466                                 QAT_LOG(ERR, "Invalid RSA key type");
 467                                 return -(EINVAL);
 468                         }
 469                 }
 470         } else {
 471                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 472                 return -(EINVAL);
 473         }
 474         return 0;
 475 }
 476
 477 int
 478 qat_asym_build_request(void *in_op,
 479                         uint8_t *out_msg,
 480                         void *op_cookie,
 481                         __rte_unused enum qat_device_gen qat_dev_gen)
 482 {
 483         struct qat_asym_session *ctx;
 484         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 485         struct rte_crypto_asym_op *asym_op = op->asym;
 486         struct icp_qat_fw_pke_request *qat_req =
 487                         (struct icp_qat_fw_pke_request *)out_msg;
 488         struct qat_asym_op_cookie *cookie =
 489                                 (struct qat_asym_op_cookie *)op_cookie;
 490         int err = 0;
 491
 492         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 493         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 494                 ctx = (struct qat_asym_session *)
 495                                 op->asym->session->sess_private_data;
 496                 if (unlikely(ctx == NULL)) {
 497                         QAT_LOG(ERR, "Session has not been created for this device");
 498                         goto error;
 499                 }
 500                 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
 501                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
 502                 if (err) {
 503                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 504                         goto error;
 505                 }
 506         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 507                 qat_fill_req_tmpl(qat_req);
 508                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
 509                                 op->asym->xform);
 510                 if (err) {
 511                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 512                         goto error;
 513                 }
 514         } else {
 515                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
 516                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 517                 goto error;
 518         }
 519
 520         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 521         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 522         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 523
 524 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 525         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 526                         sizeof(struct icp_qat_fw_pke_request));
 527 #endif
 528
 529         return 0;
 530 error:
 531
 532         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 533
 534 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 535         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 536                 sizeof(struct icp_qat_fw_pke_request));
 537 #endif
 538
 539         qat_req->output_param_count = 0;
 540         qat_req->input_param_count = 0;
 541         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 542         cookie->error |= err;
 543
 544         return 0;
 545 }
 546
 547 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
 548                 struct qat_asym_op_cookie *cookie,
 549                 struct rte_crypto_asym_xform *xform)
 550 {
 551         size_t alg_size, alg_size_in_bytes = 0;
 552         struct rte_crypto_asym_op *asym_op = rx_op->asym;
 553
 554         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 555                 rte_crypto_param n = xform->modex.modulus;
 556
 557                 alg_size = cookie->alg_size;
 558                 alg_size_in_bytes = alg_size >> 3;
 559                 uint8_t *modexp_result = asym_op->modex.result.data;
 560
 561                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 562                         rte_memcpy(modexp_result +
 563                                 (asym_op->modex.result.length -
 564                                         n.length),
 565                                 cookie->output_array[0] + alg_size_in_bytes
 566                                 - n.length, n.length
 567                                 );
 568                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 569 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 570                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
 571                                         cookie->output_array[0],
 572                                         alg_size_in_bytes);
 573
 574 #endif
 575                 }
 576         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 577                 rte_crypto_param n = xform->modinv.modulus;
 578
 579                 alg_size = cookie->alg_size;
 580                 alg_size_in_bytes = alg_size >> 3;
 581                 uint8_t *modinv_result = asym_op->modinv.result.data;
 582
 583                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 584                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 585                                 - n.length),
 586                                 cookie->output_array[0] + alg_size_in_bytes
 587                                 - n.length, n.length);
 588                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 589 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 590                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
 591                                         cookie->output_array[0],
 592                                         alg_size_in_bytes);
 593 #endif
 594                 }
 595         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 596
 597                 alg_size = cookie->alg_size;
 598                 alg_size_in_bytes = alg_size >> 3;
 599                 if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_ENCRYPT ||
 600                                 asym_op->rsa.op_type ==
 601                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 602                         if (asym_op->rsa.op_type ==
 603                                         RTE_CRYPTO_ASYM_OP_ENCRYPT) {
 604                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 605
 606                                 rte_memcpy(rsa_result,
 607                                                 cookie->output_array[0],
 608                                                 alg_size_in_bytes);
 609                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 610 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 611                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Encrypted data",
 612                                                 cookie->output_array[0],
 613                                                 alg_size_in_bytes);
 614 #endif
 615                         } else if (asym_op->rsa.op_type ==
 616                                         RTE_CRYPTO_ASYM_OP_VERIFY) {
 617                                 uint8_t *rsa_result = asym_op->rsa.cipher.data;
 618
 619                                 switch (asym_op->rsa.pad) {
 620                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 621                                         rte_memcpy(rsa_result,
 622                                                         cookie->output_array[0],
 623                                                         alg_size_in_bytes);
 624                                         rx_op->status =
 625                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 626 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 627                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 628                                                 cookie->output_array[0],
 629                                                 alg_size_in_bytes);
 630 #endif
 631                                         break;
 632                                 default:
 633                                         QAT_LOG(ERR, "Padding not supported");
 634                                         rx_op->status =
 635                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 636                                         break;
 637                                 }
 638                         }
 639                 } else {
 640                         if (asym_op->rsa.op_type ==
 641                                         RTE_CRYPTO_ASYM_OP_DECRYPT) {
 642                                 uint8_t *rsa_result = asym_op->rsa.message.data;
 643
 644                                 switch (asym_op->rsa.pad) {
 645                                 case RTE_CRYPTO_RSA_PADDING_NONE:
 646                                         rte_memcpy(rsa_result,
 647                                                 cookie->output_array[0],
 648                                                 alg_size_in_bytes);
 649                                         rx_op->status =
 650                                                 RTE_CRYPTO_OP_STATUS_SUCCESS;
 651                                         break;
 652                                 default:
 653                                         QAT_LOG(ERR, "Padding not supported");
 654                                         rx_op->status =
 655                                                 RTE_CRYPTO_OP_STATUS_ERROR;
 656                                         break;
 657                                 }
 658 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 659                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Decrypted Message",
 660                                                 rsa_result, alg_size_in_bytes);
 661 #endif
 662                         } else if (asym_op->rsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) {
 663                                 uint8_t *rsa_result = asym_op->rsa.sign.data;
 664
 665                                 rte_memcpy(rsa_result,
 666                                                 cookie->output_array[0],
 667                                                 alg_size_in_bytes);
 668                                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 669 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 670                                 QAT_DP_HEXDUMP_LOG(DEBUG, "RSA Signature",
 671                                                 cookie->output_array[0],
 672                                                 alg_size_in_bytes);
 673 #endif
 674                         }
 675                 }
 676         }
 677         qat_clear_arrays_by_alg(cookie, xform, alg_size_in_bytes);
 678 }
 679
 680 void
 681 qat_asym_process_response(void **op, uint8_t *resp,
 682                 void *op_cookie)
 683 {
 684         struct qat_asym_session *ctx;
 685         struct icp_qat_fw_pke_resp *resp_msg =
 686                         (struct icp_qat_fw_pke_resp *)resp;
 687         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
 688                         (resp_msg->opaque);
 689         struct qat_asym_op_cookie *cookie = op_cookie;
 690
 691         if (cookie->error) {
 692                 cookie->error = 0;
 693                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 694                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 695                 QAT_DP_LOG(ERR, "Cookie status returned error");
 696         } else {
 697                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 698                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 699                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 700                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 701                         QAT_DP_LOG(ERR, "Asymmetric response status"
 702                                         " returned error");
 703                 }
 704                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 705                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 706                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 707                         QAT_DP_LOG(ERR, "Asymmetric common status"
 708                                         " returned error");
 709                 }
 710         }
 711
 712         if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 713                 ctx = (struct qat_asym_session *)
 714                                 rx_op->asym->session->sess_private_data;
 715                 qat_asym_collect_response(rx_op, cookie, ctx->xform);
 716         } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 717                 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
 718         }
 719         *op = rx_op;
 720
 721 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 722         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
 723                         sizeof(struct icp_qat_fw_pke_resp));
 724 #endif
 725 }
 726
 727 int
 728 qat_asym_session_configure(struct rte_cryptodev *dev __rte_unused,
 729                 struct rte_crypto_asym_xform *xform,
 730                 struct rte_cryptodev_asym_session *sess)
 731 {
 732         struct qat_asym_session *session;
 733
 734         session = (struct qat_asym_session *) sess->sess_private_data;
 735         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 736                 if (xform->modex.exponent.length == 0 ||
 737                                 xform->modex.modulus.length == 0) {
 738                         QAT_LOG(ERR, "Invalid mod exp input parameter");
 739                         return -EINVAL;
 740                 }
 741         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 742                 if (xform->modinv.modulus.length == 0) {
 743                         QAT_LOG(ERR, "Invalid mod inv input parameter");
 744                         return -EINVAL;
 745                 }
 746         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_RSA) {
 747                 if (xform->rsa.n.length == 0) {
 748                         QAT_LOG(ERR, "Invalid rsa input parameter");
 749                         return -EINVAL;
 750                 }
 751         } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
 752                         || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
 753                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 754                 return -EINVAL;
 755         } else {
 756                 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
 757                 return -EINVAL;
 758         }
 759
 760         session->xform = xform;
 761         qat_asym_build_req_tmpl(session);
 762
 763         return 0;
 764 }
 765
 766 unsigned int qat_asym_session_get_private_size(
 767                 struct rte_cryptodev *dev __rte_unused)
 768 {
 769         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
 770 }
 771
 772 void
 773 qat_asym_session_clear(struct rte_cryptodev *dev,
 774                 struct rte_cryptodev_asym_session *sess)
 775 {
 776         void *sess_priv = sess->sess_private_data;
 777         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
 778
 779         if (sess_priv)
 780                 memset(s, 0, qat_asym_session_get_private_size(dev));
 781 }