drivers/crypto/qat/qat_asym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6
   7 #include "qat_asym.h"
   8 #include "icp_qat_fw_pke.h"
   9 #include "icp_qat_fw.h"
  10 #include "qat_pke_functionality_arrays.h"
  11
  12 #define qat_asym_sz_2param(arg) (arg, sizeof(arg)/sizeof(*arg))
  13
  14 static int qat_asym_get_sz_and_func_id(const uint32_t arr[][2],
  15                 size_t arr_sz, size_t *size, uint32_t *func_id)
  16 {
  17         size_t i;
  18
  19         for (i = 0; i < arr_sz; i++) {
  20                 if (*size <= arr[i][0]) {
  21                         *size = arr[i][0];
  22                         *func_id = arr[i][1];
  23                         return 0;
  24                 }
  25         }
  26         return -1;
  27 }
  28
  29 static void qat_asym_build_req_tmpl(void *sess_private_data,
  30                 struct rte_crypto_asym_xform *xform)
  31 {
  32
  33         struct icp_qat_fw_pke_request *qat_req;
  34         struct qat_asym_session *session = sess_private_data;
  35
  36         qat_req = &session->req_tmpl;
  37         memset(qat_req, 0, sizeof(*qat_req));
  38         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;
  39
  40         qat_req->pke_hdr.hdr_flags =
  41                         ICP_QAT_FW_COMN_HDR_FLAGS_BUILD
  42                         (ICP_QAT_FW_COMN_REQ_FLAG_SET);
  43         qat_req->pke_hdr.comn_req_flags = 0;
  44         qat_req->pke_hdr.resrvd1 = 0;
  45         qat_req->pke_hdr.resrvd2 = 0;
  46         qat_req->pke_hdr.kpt_mask = 0;
  47         qat_req->pke_hdr.kpt_rn_mask = 0;
  48         qat_req->pke_hdr.cd_pars.content_desc_addr = 0;
  49         qat_req->pke_hdr.cd_pars.content_desc_resrvd = 0;
  50         qat_req->resrvd1 = 0;
  51         qat_req->resrvd2 = 0;
  52         qat_req->next_req_adr = 0;
  53
  54         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
  55                 qat_req->output_param_count = 1;
  56                 qat_req->input_param_count = 3;
  57         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
  58                 qat_req->output_param_count = 1;
  59                 qat_req->input_param_count = 2;
  60         }
  61 }
  62
  63 static size_t max_of(int n, ...)
  64 {
  65         va_list args;
  66         size_t len = 0, num;
  67         int i;
  68
  69         va_start(args, n);
  70         len = va_arg(args, size_t);
  71
  72         for (i = 0; i < n - 1; i++) {
  73                 num = va_arg(args, size_t);
  74                 if (num > len)
  75                         len = num;
  76         }
  77         va_end(args);
  78
  79         return len;
  80 }
  81
  82 static void qat_clear_arrays(struct qat_asym_op_cookie *cookie,
  83                 int in_count, int out_count, int in_size, int out_size)
  84 {
  85         int i;
  86
  87         for (i = 0; i < in_count; i++)
  88                 memset(cookie->input_array[i], 0x0, in_size);
  89         for (i = 0; i < out_count; i++)
  90                 memset(cookie->output_array[i], 0x0, out_size);
  91 }
  92
  93 static int qat_asym_check_nonzero(rte_crypto_param n)
  94 {
  95         if (n.length < 8) {
  96                 /* Not a case for any cryptograpic function except for DH
  97                  * generator which very often can be of one byte length
  98                  */
  99                 size_t i;
 100
 101                 if (n.data[n.length - 1] == 0x0) {
 102                         for (i = 0; i < n.length - 1; i++)
 103                                 if (n.data[i] != 0x0)
 104                                         break;
 105                         if (i == n.length - 1)
 106                                 return QAT_ASYM_ERROR_INVALID_PARAM;
 107                 }
 108         } else if (*(uint64_t *)&n.data[
 109                                 n.length - 8] == 0) {
 110                 /* Very likely it is zeroed modulus */
 111                 size_t i;
 112
 113                 for (i = 0; i < n.length - 8; i++)
 114                         if (n.data[i] != 0x0)
 115                                 break;
 116                 if (i == n.length - 8)
 117                         return QAT_ASYM_ERROR_INVALID_PARAM;
 118         }
 119
 120         return 0;
 121 }
 122
 123 int
 124 qat_asym_build_request(void *in_op,
 125                         uint8_t *out_msg,
 126                         void *op_cookie,
 127                         __rte_unused enum qat_device_gen qat_dev_gen)
 128 {
 129         struct qat_asym_session *ctx;
 130         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
 131         struct rte_crypto_asym_op *asym_op = op->asym;
 132         struct icp_qat_fw_pke_request *qat_req =
 133                         (struct icp_qat_fw_pke_request *)out_msg;
 134         struct qat_asym_op_cookie *cookie =
 135                                 (struct qat_asym_op_cookie *)op_cookie;
 136
 137         uint64_t err = 0;
 138         size_t alg_size;
 139         size_t alg_size_in_bytes;
 140         uint32_t func_id;
 141
 142         ctx = (struct qat_asym_session *)get_asym_session_private_data(
 143                         op->asym->session, cryptodev_qat_asym_driver_id);
 144         rte_mov64((uint8_t *)qat_req, (const uint8_t *)&(ctx->req_tmpl));
 145         qat_req->pke_mid.opaque = (uint64_t)(uintptr_t)op;
 146
 147         qat_req->pke_mid.src_data_addr = cookie->input_addr;
 148         qat_req->pke_mid.dest_data_addr = cookie->output_addr;
 149
 150         if (ctx->alg == QAT_PKE_MODEXP) {
 151                 err = qat_asym_check_nonzero(ctx->sess_alg_params.mod_exp.n);
 152                 if (err) {
 153                         QAT_LOG(ERR, "Empty modulus in modular exponentiation,"
 154                                         " aborting this operation");
 155                         goto error;
 156                 }
 157
 158                 alg_size_in_bytes = max_of(3, asym_op->modex.base.length,
 159                                ctx->sess_alg_params.mod_exp.e.length,
 160                                ctx->sess_alg_params.mod_exp.n.length);
 161                 alg_size = alg_size_in_bytes << 3;
 162
 163                 if (qat_asym_get_sz_and_func_id(MOD_EXP_SIZE,
 164                                 sizeof(MOD_EXP_SIZE)/sizeof(*MOD_EXP_SIZE),
 165                                 &alg_size, &func_id)) {
 166                         err = QAT_ASYM_ERROR_INVALID_PARAM;
 167                         goto error;
 168                 }
 169
 170                 alg_size_in_bytes = alg_size >> 3;
 171                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 172                         asym_op->modex.base.length
 173                         , asym_op->modex.base.data,
 174                         asym_op->modex.base.length);
 175                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 176                         ctx->sess_alg_params.mod_exp.e.length
 177                         , ctx->sess_alg_params.mod_exp.e.data,
 178                         ctx->sess_alg_params.mod_exp.e.length);
 179                 rte_memcpy(cookie->input_array[2]  + alg_size_in_bytes -
 180                         ctx->sess_alg_params.mod_exp.n.length,
 181                         ctx->sess_alg_params.mod_exp.n.data,
 182                         ctx->sess_alg_params.mod_exp.n.length);
 183                 cookie->alg_size = alg_size;
 184                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 185 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 186                 QAT_DP_HEXDUMP_LOG(DEBUG, "base",
 187                                 cookie->input_array[0],
 188                                 alg_size_in_bytes);
 189                 QAT_DP_HEXDUMP_LOG(DEBUG, "exponent",
 190                                 cookie->input_array[1],
 191                                 alg_size_in_bytes);
 192                 QAT_DP_HEXDUMP_LOG(DEBUG, "modulus",
 193                                 cookie->input_array[2],
 194                                 alg_size_in_bytes);
 195 #endif
 196         } else if (ctx->alg == QAT_PKE_MODINV) {
 197                 err = qat_asym_check_nonzero(ctx->sess_alg_params.mod_inv.n);
 198                 if (err) {
 199                         QAT_LOG(ERR, "Empty modulus in modular multiplicative"
 200                                         " inverse, aborting this operation");
 201                         goto error;
 202                 }
 203
 204                 alg_size_in_bytes = max_of(2, asym_op->modinv.base.length,
 205                                 ctx->sess_alg_params.mod_inv.n.length);
 206                 alg_size = alg_size_in_bytes << 3;
 207
 208                 if (ctx->sess_alg_params.mod_inv.n.data[
 209                                 ctx->sess_alg_params.mod_inv.n.length - 1] & 0x01) {
 210                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_ODD,
 211                                         sizeof(MOD_INV_IDS_ODD)/
 212                                         sizeof(*MOD_INV_IDS_ODD),
 213                                         &alg_size, &func_id)) {
 214                                 err = QAT_ASYM_ERROR_INVALID_PARAM;
 215                                 goto error;
 216                         }
 217                 } else {
 218                         if (qat_asym_get_sz_and_func_id(MOD_INV_IDS_EVEN,
 219                                         sizeof(MOD_INV_IDS_EVEN)/
 220                                         sizeof(*MOD_INV_IDS_EVEN),
 221                                         &alg_size, &func_id)) {
 222                                 err = QAT_ASYM_ERROR_INVALID_PARAM;
 223                                 goto error;
 224                         }
 225                 }
 226
 227                 alg_size_in_bytes = alg_size >> 3;
 228                 rte_memcpy(cookie->input_array[0] + alg_size_in_bytes -
 229                         asym_op->modinv.base.length
 230                                 , asym_op->modinv.base.data,
 231                                 asym_op->modinv.base.length);
 232                 rte_memcpy(cookie->input_array[1] + alg_size_in_bytes -
 233                                 ctx->sess_alg_params.mod_inv.n.length
 234                                 , ctx->sess_alg_params.mod_inv.n.data,
 235                                 ctx->sess_alg_params.mod_inv.n.length);
 236                 cookie->alg_size = alg_size;
 237                 qat_req->pke_hdr.cd_pars.func_id = func_id;
 238 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 239                 QAT_DP_HEXDUMP_LOG(DEBUG, "base",
 240                                 cookie->input_array[0],
 241                                 alg_size_in_bytes);
 242                 QAT_DP_HEXDUMP_LOG(DEBUG, "modulus",
 243                                 cookie->input_array[1],
 244                                 alg_size_in_bytes);
 245 #endif
 246         }
 247
 248 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 249         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
 250                         sizeof(struct icp_qat_fw_pke_request));
 251 #endif
 252         return 0;
 253 error:
 254         qat_req->output_param_count = 0;
 255         qat_req->input_param_count = 0;
 256         qat_req->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 257         cookie->error |= err;
 258
 259         return 0;
 260 }
 261
 262 void
 263 qat_asym_process_response(void **op, uint8_t *resp,
 264                 void *op_cookie)
 265 {
 266         struct qat_asym_session *ctx;
 267         struct icp_qat_fw_pke_resp *resp_msg =
 268                         (struct icp_qat_fw_pke_resp *)resp;
 269         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
 270                         (resp_msg->opaque);
 271         struct rte_crypto_asym_op *asym_op = rx_op->asym;
 272         struct qat_asym_op_cookie *cookie = op_cookie;
 273         size_t alg_size, alg_size_in_bytes;
 274
 275         ctx = (struct qat_asym_session *)get_asym_session_private_data(
 276                         rx_op->asym->session, cryptodev_qat_asym_driver_id);
 277
 278         *op = rx_op;
 279         rx_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 280
 281         if (cookie->error) {
 282                 cookie->error = 0;
 283                 rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 284                 QAT_DP_LOG(ERR, "Cookie status returned error");
 285         } else {
 286                 if (ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(
 287                         resp_msg->pke_resp_hdr.resp_status.pke_resp_flags)) {
 288                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 289                         QAT_DP_LOG(ERR, "Asymmetric response status returned error");
 290                 }
 291                 if (resp_msg->pke_resp_hdr.resp_status.comn_err_code) {
 292                         rx_op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 293                         QAT_DP_LOG(ERR, "Asymmetric common status returned error");
 294                 }
 295         }
 296
 297         if (ctx->alg == QAT_PKE_MODEXP) {
 298                 alg_size = cookie->alg_size;
 299                 alg_size_in_bytes = alg_size >> 3;
 300                 uint8_t *modexp_result = asym_op->modex.result.data;
 301
 302                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 303                         rte_memcpy(modexp_result +
 304                                 (asym_op->modex.result.length -
 305                                         ctx->sess_alg_params.mod_exp.n.length),
 306                                 cookie->output_array[0] + alg_size_in_bytes
 307                                 - ctx->sess_alg_params.mod_exp.n.length,
 308                                 ctx->sess_alg_params.mod_exp.n.length
 309                                 );
 310                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 311 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 312                         QAT_DP_HEXDUMP_LOG(DEBUG, "modexp_result",
 313                                         cookie->output_array[0],
 314                                         alg_size_in_bytes);
 315 #endif
 316                 }
 317                 qat_clear_arrays(cookie, 3, 1, alg_size_in_bytes,
 318                                 alg_size_in_bytes);
 319         } else if (ctx->alg == QAT_PKE_MODINV) {
 320                 alg_size = cookie->alg_size;
 321                 alg_size_in_bytes = alg_size >> 3;
 322                 uint8_t *modinv_result = asym_op->modinv.result.data;
 323
 324                 if (rx_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) {
 325                         rte_memcpy(modinv_result + (asym_op->modinv.result.length
 326                                 - ctx->sess_alg_params.mod_inv.n.length),
 327                                 cookie->output_array[0] + alg_size_in_bytes
 328                                 - ctx->sess_alg_params.mod_inv.n.length,
 329                                 ctx->sess_alg_params.mod_inv.n.length);
 330                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 331 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 332                         QAT_DP_HEXDUMP_LOG(DEBUG, "modinv_result",
 333                                         cookie->output_array[0],
 334                                         alg_size_in_bytes);
 335 #endif
 336                 }
 337                 qat_clear_arrays(cookie, 2, 1, alg_size_in_bytes,
 338                         alg_size_in_bytes);
 339         }
 340
 341 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 342         QAT_DP_HEXDUMP_LOG(DEBUG, "resp_msg:", resp_msg,
 343                         sizeof(struct icp_qat_fw_pke_resp));
 344 #endif
 345 }
 346
 347 int
 348 qat_asym_session_configure(struct rte_cryptodev *dev,
 349                 struct rte_crypto_asym_xform *xform,
 350                 struct rte_cryptodev_asym_session *sess,
 351                 struct rte_mempool *mempool)
 352 {
 353         int err = 0;
 354         void *sess_private_data;
 355         struct qat_asym_session *session;
 356
 357         if (rte_mempool_get(mempool, &sess_private_data)) {
 358                 QAT_LOG(ERR,
 359                         "Couldn't get object from session mempool");
 360                 return -ENOMEM;
 361         }
 362
 363         session = sess_private_data;
 364         if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODEX) {
 365                 session->sess_alg_params.mod_exp.e = xform->modex.exponent;
 366                 session->sess_alg_params.mod_exp.n = xform->modex.modulus;
 367                 session->alg = QAT_PKE_MODEXP;
 368
 369                 if (xform->modex.exponent.length == 0 ||
 370                                 xform->modex.modulus.length == 0) {
 371                         QAT_LOG(ERR, "Invalid mod exp input parameter");
 372                         err = -EINVAL;
 373                         goto error;
 374                 }
 375         } else if (xform->xform_type == RTE_CRYPTO_ASYM_XFORM_MODINV) {
 376                 session->sess_alg_params.mod_inv.n = xform->modinv.modulus;
 377                 session->alg = QAT_PKE_MODINV;
 378
 379                 if (xform->modinv.modulus.length == 0) {
 380                         QAT_LOG(ERR, "Invalid mod inv input parameter");
 381                         err = -EINVAL;
 382                         goto error;
 383                 }
 384         } else {
 385                 QAT_LOG(ERR, "Invalid asymmetric crypto xform");
 386                 err = -EINVAL;
 387                 goto error;
 388         }
 389         qat_asym_build_req_tmpl(sess_private_data, xform);
 390         set_asym_session_private_data(sess, dev->driver_id,
 391                 sess_private_data);
 392
 393         return 0;
 394 error:
 395         rte_mempool_put(mempool, sess_private_data);
 396         return err;
 397 }
 398
 399 unsigned int qat_asym_session_get_private_size(
 400                 struct rte_cryptodev *dev __rte_unused)
 401 {
 402         return RTE_ALIGN_CEIL(sizeof(struct qat_asym_session), 8);
 403 }
 404
 405 void
 406 qat_asym_session_clear(struct rte_cryptodev *dev,
 407                 struct rte_cryptodev_asym_session *sess)
 408 {
 409         uint8_t index = dev->driver_id;
 410         void *sess_priv = get_asym_session_private_data(sess, index);
 411         struct qat_asym_session *s = (struct qat_asym_session *)sess_priv;
 412
 413         if (sess_priv) {
 414                 memset(s, 0, qat_asym_session_get_private_size(dev));
 415                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 416
 417                 set_asym_session_private_data(sess, index, NULL);
 418                 rte_mempool_put(sess_mp, sess_priv);
 419         }
 420 }