drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include "qat_asym.h"
   8 #include "icp_qat_fw_pke.h"
   9 #include "icp_qat_fw.h"
  10 #include "qat_pke_functionality_arrays.h"
  11
  12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
  13
  14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
  15                 size_t arr_sz, size_t *size, uint32_t *func_id)
  16 {
  17         size_t i;
  18
  19         for (i = 0; i < arr_sz; i++) {
  20                 if (*size <= arr[i][0]) {
  21                         *size = arr[i][0];
  22                         *func_id = arr[i][1];
  23                         return 0;
  24                 }
  25         }
  26         return -1;
  27 }
  28
  29 static inline void qat_fill_req_tmpl(struct icp_qat_fw_pke_request *qat_req)
  30 {
  31         memset(qat_req, 0, sizeof(*qat_req));
  32         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
  33
  34         qat_req->pke_hdr.hdr_flags =
  35                         ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
  36                         (ICP_QAT_FW_COMN_REQ_FLAG_SET);
  37 }
  38
  39 static inline void qat_asym_build_req_tmpl(void *sess_private_data)
  40 {
  41         struct icp_qat_fw_pke_request *qat_req;
  42         struct qat_asym_session *session = sess_private_data;
  43
  44         qat_req = &session->req_tmpl;
  45         qat_fill_req_tmpl(qat_req);
  46 }
  47
  48 static size_t max_of(int n, ...)
  49 {
  50         va_list args;
  51         size_t len = 0, num;
  52         int i;
  53
  54         va_start(args, n);
  55         len = va_arg(args, size_t);
  56
  57         for (i = 0; i < n - 1; i++) {
  58                 num = va_arg(args, size_t);
  59                 if (num > len)
  60                         len = num;
  61         }
  62         va_end(args);
  63
  64         return len;
  65 }
  66
  67 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
  68                 int in_count, int out_count, int in_size, int out_size)
  69 {
  70         int i;
  71
  72         for (i = 0; i < in_count; i++)
  73                 memset(cookie->input_array[i], 0x0, in_size);
  74         for (i = 0; i < out_count; i++)
  75                 memset(cookie->output_array[i], 0x0, out_size);
  76 }
  77
  78 static void qat_clear_arrays_by_alg(struct qat_asym_op_cookie *cookie,
  79                 enum rte_crypto_asym_xform_type alg, int in_size, int out_size)
  80 {
  81         if (alg == RTE_CRYPTO_ASYM_XFORM_MODEX)
  82                 qat_clear_arrays(cookie, QAT_ASYM_MODEXP_NUM_IN_PARAMS,
  83                                 QAT_ASYM_MODEXP_NUM_OUT_PARAMS, in_size,
  84                                 out_size);
  85         else if (alg == RTE_CRYPTO_ASYM_XFORM_MODINV)
  86                 qat_clear_arrays(cookie, QAT_ASYM_MODINV_NUM_IN_PARAMS,
  87                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS, in_size,
  88                                 out_size);
  89 }
  90
  91 static int qat_asym_check_nonzero(rte_crypto_param n)
  92 {
  93         if (n.length < 8) {
  94                 /* Not a case for any cryptograpic function except for DH
  95                  * generator which very often can be of one byte length
  96                  */
  97                 size_t i;
  98
  99                 if (n.data[n.length - 1] == 0x0) {
 100                         for (i = 0; i < n.length - 1; i++)
 101                                 if (n.data[i] != 0x0)
 102                                         break;
 103                         if (i == n.length - 1)
 104                                 return -(EINVAL);
 105                 }
 106         } else if (*(uint64_t *)&n.data[
 107                                 n.length - 8] == 0) {
 108                 /* Very likely it is zeroed modulus */
 109                 size_t i;
 110
 111                 for (i = 0; i < n.length - 8; i++)
 112                         if (n.data[i] != 0x0)
 113                                 break;
 114                 if (i == n.length - 8)
 115                         return -(EINVAL);
 116         }
 117
 118         return 0;
 119 }
 120
 121 static int
 122 qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 123                 struct icp_qat_fw_pke_request *qat_req,
 124                 struct qat_asym_op_cookie *cookie,
 125                 struct rte_crypto_asym_xform *xform)
 126 {
 127         int err = 0;
 128         size_t alg_size;
 129         size_t alg_size_in_bytes;
 130         uint32_t func_id = 0;
 131
 132         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 133                 err = qat_asym_check_nonzero(xform->modex.modulus);
 134                 if (err) {
 135                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
 136                                         " aborting this operation");
 137                         return err;
 138                 }
 139
 140                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
 141                                xform->modex.exponent.length,
 142                                xform->modex.modulus.length);
 143                 alg_size = alg_size_in_bytes << 3;
 144
 145                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
 146                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
 147                                 &alg_size, &func_id)) {
 148                         return -(EINVAL);
 149                 }
 150
 151                 alg_size_in_bytes = alg_size >> 3;
 152                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 153                         asym_op->modex.base.length
 154                         , asym_op->modex.base.data,
 155                         asym_op->modex.base.length);
 156                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 157                         xform->modex.exponent.length
 158                         , xform->modex.exponent.data,
 159                         xform->modex.exponent.length);
 160                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
 161                         xform->modex.modulus.length,
 162                         xform->modex.modulus.data,
 163                         xform->modex.modulus.length);
 164                 cookie->alg_size = alg_size;
 165                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 166                 qat_req->input_param_count = QAT_ASYM_MODEXP_NUM_IN_PARAMS;
 167                 qat_req->output_param_count = QAT_ASYM_MODEXP_NUM_OUT_PARAMS;
 168 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 169                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp base",
 170                                 cookie->input_array[0],
 171                                 alg_size_in_bytes);
 172                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp exponent",
 173                                 cookie->input_array[1],
 174                                 alg_size_in_bytes);
 175                 QAT_DP_HEXDUMP_LOG(DEBUG, " ModExpmodulus",
 176                                 cookie->input_array[2],
 177                                 alg_size_in_bytes);
 178 #endif
 179         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 180                 err = qat_asym_check_nonzero(xform->modinv.modulus);
 181                 if (err) {
 182                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
 183                                         " inverse, aborting this operation");
 184                         return err;
 185                 }
 186
 187                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
 188                                 xform->modinv.modulus.length);
 189                 alg_size = alg_size_in_bytes << 3;
 190
 191                 if (xform->modinv.modulus.data[
 192                                 xform->modinv.modulus.length - 1] & 0x01) {
 193                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
 194                                         sizeof(MOD_INV_IDS_ODD)/
 195                                         sizeof(*MOD_INV_IDS_ODD),
 196                                         &alg_size, &func_id)) {
 197                                 return -(EINVAL);
 198                         }
 199                 } else {
 200                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
 201                                         sizeof(MOD_INV_IDS_EVEN)/
 202                                         sizeof(*MOD_INV_IDS_EVEN),
 203                                         &alg_size, &func_id)) {
 204                                 return -(EINVAL);
 205                         }
 206                 }
 207
 208                 alg_size_in_bytes = alg_size >> 3;
 209                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 210                         asym_op->modinv.base.length
 211                                 , asym_op->modinv.base.data,
 212                                 asym_op->modinv.base.length);
 213                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 214                                 xform->modinv.modulus.length
 215                                 , xform->modinv.modulus.data,
 216                                 xform->modinv.modulus.length);
 217                 cookie->alg_size = alg_size;
 218                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 219                 qat_req->input_param_count =
 220                                 QAT_ASYM_MODINV_NUM_IN_PARAMS;
 221                 qat_req->output_param_count =
 222                                 QAT_ASYM_MODINV_NUM_OUT_PARAMS;
 223 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 224                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv base",
 225                                 cookie->input_array[0],
 226                                 alg_size_in_bytes);
 227                 QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv modulus",
 228                                 cookie->input_array[1],
 229                                 alg_size_in_bytes);
 230 #endif
 231         } else {
 232                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 233                 return -(EINVAL);
 234         }
 235         return 0;
 236 }
 237
 238 int
 239 qat_asym_build_request(void *in_op,
 240                         uint8_t *out_msg,
 241                         void *op_cookie,
 242                         __rte_unused enum qat_device_gen qat_dev_gen)
 243 {
 244         struct qat_asym_session *ctx;
 245         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 246         struct rte_crypto_asym_op *asym_op = op->asym;
 247         struct icp_qat_fw_pke_request *qat_req =
 248                         (struct icp_qat_fw_pke_request *)out_msg;
 249         struct qat_asym_op_cookie *cookie =
 250                                 (struct qat_asym_op_cookie *)op_cookie;
 251         int err = 0;
 252
 253         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 254         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 255                 ctx = (struct qat_asym_session *)
 256                         get_asym_session_private_data(
 257                         op->asym->session, cryptodev_qat_asym_driver_id);
 258                 if (unlikely(ctx == NULL)) {
 259                         QAT_LOG(ERR, "Session has not been created for this device");
 260                         goto error;
 261                 }
 262                 rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
 263                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie, ctx->xform);
 264                 if (err) {
 265                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 266                         goto error;
 267                 }
 268         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 269                 qat_fill_req_tmpl(qat_req);
 270                 err = qat_asym_fill_arrays(asym_op, qat_req, cookie,
 271                                 op->asym->xform);
 272                 if (err) {
 273                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 274                         goto error;
 275                 }
 276         } else {
 277                 QAT_DP_LOG(ERR, "Invalid session/xform settings");
 278                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 279                 goto error;
 280         }
 281
 282         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 283         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 284         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 285
 286 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 287         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 288                         sizeof(struct icp_qat_fw_pke_request));
 289 #endif
 290
 291         return 0;
 292 error:
 293
 294         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 295
 296 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 297         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 298                 sizeof(struct icp_qat_fw_pke_request));
 299 #endif
 300
 301         qat_req->output_param_count = 0;
 302         qat_req->input_param_count = 0;
 303         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 304         cookie->error |= err;
 305
 306         return 0;
 307 }
 308
 309 static void qat_asym_collect_response(struct rte_crypto_op *rx_op,
 310                 struct qat_asym_op_cookie *cookie,
 311                 struct rte_crypto_asym_xform *xform)
 312 {
 313         size_t alg_size, alg_size_in_bytes = 0;
 314         struct rte_crypto_asym_op *asym_op = rx_op->asym;
 315
 316         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 317                 rte_crypto_param n = xform->modex.modulus;
 318
 319                 alg_size = cookie->alg_size;
 320                 alg_size_in_bytes = alg_size >> 3;
 321                 uint8_t *modexp_result = asym_op->modex.result.data;
 322
 323                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 324                         rte_memcpy(modexp_result +
 325                                 (asym_op->modex.result.length -
 326                                         n.length),
 327                                 cookie->output_array[0] + alg_size_in_bytes
 328                                 - n.length, n.length
 329                                 );
 330                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 331 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 332                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModExp result",
 333                                         cookie->output_array[0],
 334                                         alg_size_in_bytes);
 335
 336 #endif
 337                 }
 338         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 339                 rte_crypto_param n = xform->modinv.modulus;
 340
 341                 alg_size = cookie->alg_size;
 342                 alg_size_in_bytes = alg_size >> 3;
 343                 uint8_t *modinv_result = asym_op->modinv.result.data;
 344
 345                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 346                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 347                                 - n.length),
 348                                 cookie->output_array[0] + alg_size_in_bytes
 349                                 - n.length, n.length);
 350                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 351 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 352                         QAT_DP_HEXDUMP_LOG(DEBUG, "ModInv result",
 353                                         cookie->output_array[0],
 354                                         alg_size_in_bytes);
 355 #endif
 356                 }
 357         }
 358         qat_clear_arrays_by_alg(cookie, xform->xform_type, alg_size_in_bytes,
 359                         alg_size_in_bytes);
 360 }
 361
 362 void
 363 qat_asym_process_response(void **op, uint8_t *resp,
 364                 void *op_cookie)
 365 {
 366         struct qat_asym_session *ctx;
 367         struct icp_qat_fw_pke_resp *resp_msg =
 368                         (struct icp_qat_fw_pke_resp *)resp;
 369         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
 370                         (resp_msg->opaque);
 371         struct qat_asym_op_cookie *cookie = op_cookie;
 372
 373         if (cookie->error) {
 374                 cookie->error = 0;
 375                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 376                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 377                 QAT_DP_LOG(ERR, "Cookie status returned error");
 378         } else {
 379                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 380                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 381                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 382                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 383                         QAT_DP_LOG(ERR, "Asymmetric response status"
 384                                         " returned error");
 385                 }
 386                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 387                         if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 388                                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 389                         QAT_DP_LOG(ERR, "Asymmetric common status"
 390                                         " returned error");
 391                 }
 392         }
 393
 394         if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 395                 ctx = (struct qat_asym_session *)get_asym_session_private_data(
 396                                 rx_op->asym->session, cryptodev_qat_asym_driver_id);
 397                 qat_asym_collect_response(rx_op, cookie, ctx->xform);
 398         } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 399                 qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform);
 400         }
 401         *op = rx_op;
 402
 403 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 404         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
 405                         sizeof(struct icp_qat_fw_pke_resp));
 406 #endif
 407 }
 408
 409 int
 410 qat_asym_session_configure(struct rte_cryptodev *dev,
 411                 struct rte_crypto_asym_xform *xform,
 412                 struct rte_cryptodev_asym_session *sess,
 413                 struct rte_mempool *mempool)
 414 {
 415         int err = 0;
 416         void *sess_private_data;
 417         struct qat_asym_session *session;
 418
 419         if (rte_mempool_get(mempool, &sess_private_data)) {
 420                 QAT_LOG(ERR,
 421                         "Couldn't get object from session mempool");
 422                 return -ENOMEM;
 423         }
 424
 425         session = sess_private_data;
 426         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 427                 if (xform->modex.exponent.length == 0 ||
 428                                 xform->modex.modulus.length == 0) {
 429                         QAT_LOG(ERR, "Invalid mod exp input parameter");
 430                         err = -EINVAL;
 431                         goto error;
 432                 }
 433         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 434                 if (xform->modinv.modulus.length == 0) {
 435                         QAT_LOG(ERR, "Invalid mod inv input parameter");
 436                         err = -EINVAL;
 437                         goto error;
 438                 }
 439         } else if (xform->xform_type >= RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
 440                         || xform->xform_type <= RTE_CRYPTO_ASYM_XFORM_NONE) {
 441                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 442                 err = -EINVAL;
 443                 goto error;
 444         } else {
 445                 QAT_LOG(ERR, "Asymmetric crypto xform not implemented");
 446                 err = -EINVAL;
 447                 goto error;
 448         }
 449
 450         session->xform = xform;
 451         qat_asym_build_req_tmpl(sess_private_data);
 452         set_asym_session_private_data(sess, dev->driver_id,
 453                 sess_private_data);
 454
 455         return 0;
 456 error:
 457         rte_mempool_put(mempool, sess_private_data);
 458         return err;
 459 }
 460
 461 unsigned int qat_asym_session_get_private_size(
 462                 struct rte_cryptodev *dev __rte_unused)
 463 {
 464         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
 465 }
 466
 467 void
 468 qat_asym_session_clear(struct rte_cryptodev *dev,
 469                 struct rte_cryptodev_asym_session *sess)
 470 {
 471         uint8_t index = dev->driver_id;
 472         void *sess_priv = get_asym_session_private_data(sess, index);
 473         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
 474
 475         if (sess_priv) {
 476                 memset(s, 0, qat_asym_session_get_private_size(dev));
 477                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 478
 479                 set_asym_session_private_data(sess, index, NULL);
 480                 rte_mempool_put(sess_mp, sess_priv);
 481         }
 482 }