drivers/crypto/openssl/rte_openssl_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_hexdump.h>
   7 #include <rte_cryptodev.h>
   8 #include <cryptodev_pmd.h>
   9 #include <rte_bus_vdev.h>
  10 #include <rte_malloc.h>
  11 #include <rte_cpuflags.h>
  12
  13 #include <openssl/hmac.h>
  14 #include <openssl/evp.h>
  15
  16 #include "openssl_pmd_private.h"
  17 #include "compat.h"
  18
  19 #define DES_BLOCK_SIZE 8
  20
  21 static uint8_t cryptodev_driver_id;
  22
  23 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  24 static HMAC_CTX *HMAC_CTX_new(void)
  25 {
  26         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  27
  28         if (ctx != NULL)
  29                 HMAC_CTX_init(ctx);
  30         return ctx;
  31 }
  32
  33 static void HMAC_CTX_free(HMAC_CTX *ctx)
  34 {
  35         if (ctx != NULL) {
  36                 HMAC_CTX_cleanup(ctx);
  37                 OPENSSL_free(ctx);
  38         }
  39 }
  40 #endif
  41
  42 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
  43
  44 #include <openssl/provider.h>
  45 #include <openssl/core_names.h>
  46 #include <openssl/param_build.h>
  47
  48 #define MAX_OSSL_ALGO_NAME_SIZE         16
  49
  50 OSSL_PROVIDER *legacy;
  51 OSSL_PROVIDER *deflt;
  52
  53 static void ossl_legacy_provider_load(void)
  54 {
  55         /* Load Multiple providers into the default (NULL) library context */
  56         legacy = OSSL_PROVIDER_load(NULL, "legacy");
  57         if (legacy == NULL) {
  58                 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n");
  59                 return;
  60         }
  61
  62         deflt = OSSL_PROVIDER_load(NULL, "default");
  63         if (deflt == NULL) {
  64                 OPENSSL_LOG(ERR, "Failed to load Default provider\n");
  65                 OSSL_PROVIDER_unload(legacy);
  66                 return;
  67         }
  68 }
  69
  70 static void ossl_legacy_provider_unload(void)
  71 {
  72         OSSL_PROVIDER_unload(legacy);
  73         OSSL_PROVIDER_unload(deflt);
  74 }
  75
  76 static __rte_always_inline const char *
  77 digest_name_get(enum rte_crypto_auth_algorithm algo)
  78 {
  79         switch (algo) {
  80         case RTE_CRYPTO_AUTH_MD5_HMAC:
  81                 return OSSL_DIGEST_NAME_MD5;
  82         case RTE_CRYPTO_AUTH_SHA1_HMAC:
  83                 return OSSL_DIGEST_NAME_SHA1;
  84         case RTE_CRYPTO_AUTH_SHA224_HMAC:
  85                 return OSSL_DIGEST_NAME_SHA2_224;
  86         case RTE_CRYPTO_AUTH_SHA256_HMAC:
  87                 return OSSL_DIGEST_NAME_SHA2_256;
  88         case RTE_CRYPTO_AUTH_SHA384_HMAC:
  89                 return OSSL_DIGEST_NAME_SHA2_384;
  90         case RTE_CRYPTO_AUTH_SHA512_HMAC:
  91                 return OSSL_DIGEST_NAME_SHA2_512;
  92         default:
  93                 return NULL;
  94         }
  95 }
  96 #endif
  97
  98 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  99
 100 /*----------------------------------------------------------------------------*/
 101
 102 /**
 103  * Increment counter by 1
 104  * Counter is 64 bit array, big-endian
 105  */
 106 static void
 107 ctr_inc(uint8_t *ctr)
 108 {
 109         uint64_t *ctr64 = (uint64_t *)ctr;
 110
 111         *ctr64 = __builtin_bswap64(*ctr64);
 112         (*ctr64)++;
 113         *ctr64 = __builtin_bswap64(*ctr64);
 114 }
 115
 116 /*
 117  *------------------------------------------------------------------------------
 118  * Session Prepare
 119  *------------------------------------------------------------------------------
 120  */
 121
 122 /** Get xform chain order */
 123 static enum openssl_chain_order
 124 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
 125 {
 126         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
 127
 128         if (xform != NULL) {
 129                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 130                         if (xform->next == NULL)
 131                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
 132                         else if (xform->next->type ==
 133                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
 134                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
 135                 }
 136                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 137                         if (xform->next == NULL)
 138                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
 139                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 140                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
 141                 }
 142                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 143                         res = OPENSSL_CHAIN_COMBINED;
 144         }
 145
 146         return res;
 147 }
 148
 149 /** Get session cipher key from input cipher key */
 150 static void
 151 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key)
 152 {
 153         memcpy(session_key, input_key, keylen);
 154 }
 155
 156 /** Get key ede 24 bytes standard from input key */
 157 static int
 158 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede)
 159 {
 160         int res = 0;
 161
 162         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 163         switch (keylen) {
 164         case 24:
 165                 memcpy(key_ede, key, 24);
 166                 break;
 167         case 16:
 168                 /* K3 = K1 */
 169                 memcpy(key_ede, key, 16);
 170                 memcpy(key_ede + 16, key, 8);
 171                 break;
 172         case 8:
 173                 /* K1 = K2 = K3 (DES compatibility) */
 174                 memcpy(key_ede, key, 8);
 175                 memcpy(key_ede + 8, key, 8);
 176                 memcpy(key_ede + 16, key, 8);
 177                 break;
 178         default:
 179                 OPENSSL_LOG(ERR, "Unsupported key size");
 180                 res = -EINVAL;
 181         }
 182
 183         return res;
 184 }
 185
 186 /** Get adequate openssl function for input cipher algorithm */
 187 static uint8_t
 188 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 189                 const EVP_CIPHER **algo)
 190 {
 191         int res = 0;
 192
 193         if (algo != NULL) {
 194                 switch (sess_algo) {
 195                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 196                         switch (keylen) {
 197                         case 8:
 198                                 *algo = EVP_des_cbc();
 199                                 break;
 200                         case 16:
 201                                 *algo = EVP_des_ede_cbc();
 202                                 break;
 203                         case 24:
 204                                 *algo = EVP_des_ede3_cbc();
 205                                 break;
 206                         default:
 207                                 res = -EINVAL;
 208                         }
 209                         break;
 210                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 211                         break;
 212                 case RTE_CRYPTO_CIPHER_AES_CBC:
 213                         switch (keylen) {
 214                         case 16:
 215                                 *algo = EVP_aes_128_cbc();
 216                                 break;
 217                         case 24:
 218                                 *algo = EVP_aes_192_cbc();
 219                                 break;
 220                         case 32:
 221                                 *algo = EVP_aes_256_cbc();
 222                                 break;
 223                         default:
 224                                 res = -EINVAL;
 225                         }
 226                         break;
 227                 case RTE_CRYPTO_CIPHER_AES_CTR:
 228                         switch (keylen) {
 229                         case 16:
 230                                 *algo = EVP_aes_128_ctr();
 231                                 break;
 232                         case 24:
 233                                 *algo = EVP_aes_192_ctr();
 234                                 break;
 235                         case 32:
 236                                 *algo = EVP_aes_256_ctr();
 237                                 break;
 238                         default:
 239                                 res = -EINVAL;
 240                         }
 241                         break;
 242                 default:
 243                         res = -EINVAL;
 244                         break;
 245                 }
 246         } else {
 247                 res = -EINVAL;
 248         }
 249
 250         return res;
 251 }
 252
 253 /** Get adequate openssl function for input auth algorithm */
 254 static uint8_t
 255 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 256                 const EVP_MD **algo)
 257 {
 258         int res = 0;
 259
 260         if (algo != NULL) {
 261                 switch (sessalgo) {
 262                 case RTE_CRYPTO_AUTH_MD5:
 263                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 264                         *algo = EVP_md5();
 265                         break;
 266                 case RTE_CRYPTO_AUTH_SHA1:
 267                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 268                         *algo = EVP_sha1();
 269                         break;
 270                 case RTE_CRYPTO_AUTH_SHA224:
 271                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 272                         *algo = EVP_sha224();
 273                         break;
 274                 case RTE_CRYPTO_AUTH_SHA256:
 275                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 276                         *algo = EVP_sha256();
 277                         break;
 278                 case RTE_CRYPTO_AUTH_SHA384:
 279                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 280                         *algo = EVP_sha384();
 281                         break;
 282                 case RTE_CRYPTO_AUTH_SHA512:
 283                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 284                         *algo = EVP_sha512();
 285                         break;
 286                 default:
 287                         res = -EINVAL;
 288                         break;
 289                 }
 290         } else {
 291                 res = -EINVAL;
 292         }
 293
 294         return res;
 295 }
 296
 297 /** Get adequate openssl function for input cipher algorithm */
 298 static uint8_t
 299 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 300                 const EVP_CIPHER **algo)
 301 {
 302         int res = 0;
 303
 304         if (algo != NULL) {
 305                 switch (sess_algo) {
 306                 case RTE_CRYPTO_AEAD_AES_GCM:
 307                         switch (keylen) {
 308                         case 16:
 309                                 *algo = EVP_aes_128_gcm();
 310                                 break;
 311                         case 24:
 312                                 *algo = EVP_aes_192_gcm();
 313                                 break;
 314                         case 32:
 315                                 *algo = EVP_aes_256_gcm();
 316                                 break;
 317                         default:
 318                                 res = -EINVAL;
 319                         }
 320                         break;
 321                 case RTE_CRYPTO_AEAD_AES_CCM:
 322                         switch (keylen) {
 323                         case 16:
 324                                 *algo = EVP_aes_128_ccm();
 325                                 break;
 326                         case 24:
 327                                 *algo = EVP_aes_192_ccm();
 328                                 break;
 329                         case 32:
 330                                 *algo = EVP_aes_256_ccm();
 331                                 break;
 332                         default:
 333                                 res = -EINVAL;
 334                         }
 335                         break;
 336                 default:
 337                         res = -EINVAL;
 338                         break;
 339                 }
 340         } else {
 341                 res = -EINVAL;
 342         }
 343
 344         return res;
 345 }
 346
 347 /* Set session AEAD encryption parameters */
 348 static int
 349 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
 350                 enum rte_crypto_aead_algorithm algo,
 351                 uint8_t tag_len, const uint8_t *key)
 352 {
 353         int iv_type = 0;
 354         unsigned int do_ccm;
 355
 356         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 357         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 358
 359         /* Select AEAD algo */
 360         switch (algo) {
 361         case RTE_CRYPTO_AEAD_AES_GCM:
 362                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 363                 if (tag_len != 16)
 364                         return -EINVAL;
 365                 do_ccm = 0;
 366                 break;
 367         case RTE_CRYPTO_AEAD_AES_CCM:
 368                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 369                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 370                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 371                         return -EINVAL;
 372                 do_ccm = 1;
 373                 break;
 374         default:
 375                 return -ENOTSUP;
 376         }
 377
 378         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 379         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 380
 381         if (get_aead_algo(algo, sess->cipher.key.length,
 382                         &sess->cipher.evp_algo) != 0)
 383                 return -EINVAL;
 384
 385         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 386
 387         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 388
 389         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 390                         NULL, NULL, NULL) <= 0)
 391                 return -EINVAL;
 392
 393         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
 394                         NULL) <= 0)
 395                 return -EINVAL;
 396
 397         if (do_ccm)
 398                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 399                                 tag_len, NULL);
 400
 401         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 402                 return -EINVAL;
 403
 404         return 0;
 405 }
 406
 407 /* Set session AEAD decryption parameters */
 408 static int
 409 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
 410                 enum rte_crypto_aead_algorithm algo,
 411                 uint8_t tag_len, const uint8_t *key)
 412 {
 413         int iv_type = 0;
 414         unsigned int do_ccm = 0;
 415
 416         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 417         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 418
 419         /* Select AEAD algo */
 420         switch (algo) {
 421         case RTE_CRYPTO_AEAD_AES_GCM:
 422                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 423                 if (tag_len != 16)
 424                         return -EINVAL;
 425                 break;
 426         case RTE_CRYPTO_AEAD_AES_CCM:
 427                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 428                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 429                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 430                         return -EINVAL;
 431                 do_ccm = 1;
 432                 break;
 433         default:
 434                 return -ENOTSUP;
 435         }
 436
 437         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 438         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 439
 440         if (get_aead_algo(algo, sess->cipher.key.length,
 441                         &sess->cipher.evp_algo) != 0)
 442                 return -EINVAL;
 443
 444         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 445
 446         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 447
 448         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 449                         NULL, NULL, NULL) <= 0)
 450                 return -EINVAL;
 451
 452         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
 453                         sess->iv.length, NULL) <= 0)
 454                 return -EINVAL;
 455
 456         if (do_ccm)
 457                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 458                                 tag_len, NULL);
 459
 460         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 461                 return -EINVAL;
 462
 463         return 0;
 464 }
 465
 466 /** Set session cipher parameters */
 467 static int
 468 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 469                 const struct rte_crypto_sym_xform *xform)
 470 {
 471         /* Select cipher direction */
 472         sess->cipher.direction = xform->cipher.op;
 473         /* Select cipher key */
 474         sess->cipher.key.length = xform->cipher.key.length;
 475
 476         /* Set IV parameters */
 477         sess->iv.offset = xform->cipher.iv.offset;
 478         sess->iv.length = xform->cipher.iv.length;
 479
 480         /* Select cipher algo */
 481         switch (xform->cipher.algo) {
 482         case RTE_CRYPTO_CIPHER_3DES_CBC:
 483         case RTE_CRYPTO_CIPHER_AES_CBC:
 484         case RTE_CRYPTO_CIPHER_AES_CTR:
 485                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 486                 sess->cipher.algo = xform->cipher.algo;
 487                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 488
 489                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 490                                 &sess->cipher.evp_algo) != 0)
 491                         return -EINVAL;
 492
 493                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 494                         sess->cipher.key.data);
 495                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 496                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 497                                         sess->cipher.evp_algo,
 498                                         NULL, xform->cipher.key.data,
 499                                         NULL) != 1) {
 500                                 return -EINVAL;
 501                         }
 502                 } else if (sess->cipher.direction ==
 503                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 504                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 505                                         sess->cipher.evp_algo,
 506                                         NULL, xform->cipher.key.data,
 507                                         NULL) != 1) {
 508                                 return -EINVAL;
 509                         }
 510                 }
 511
 512                 break;
 513
 514         case RTE_CRYPTO_CIPHER_3DES_CTR:
 515                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 516                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 517
 518                 if (get_cipher_key_ede(xform->cipher.key.data,
 519                                 sess->cipher.key.length,
 520                                 sess->cipher.key.data) != 0)
 521                         return -EINVAL;
 522                 break;
 523
 524         case RTE_CRYPTO_CIPHER_DES_CBC:
 525                 sess->cipher.algo = xform->cipher.algo;
 526                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 527                 sess->cipher.evp_algo = EVP_des_cbc();
 528
 529                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 530                         sess->cipher.key.data);
 531                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 532                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 533                                         sess->cipher.evp_algo,
 534                                         NULL, xform->cipher.key.data,
 535                                         NULL) != 1) {
 536                                 return -EINVAL;
 537                         }
 538                 } else if (sess->cipher.direction ==
 539                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 540                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 541                                         sess->cipher.evp_algo,
 542                                         NULL, xform->cipher.key.data,
 543                                         NULL) != 1) {
 544                                 return -EINVAL;
 545                         }
 546                 }
 547
 548                 break;
 549
 550         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 551                 sess->cipher.algo = xform->cipher.algo;
 552                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 553                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 554                 sess->cipher.evp_algo = EVP_des_cbc();
 555
 556                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 557                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 558                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 559                                 NULL, xform->cipher.key.data, 0) != 1)
 560                         return -EINVAL;
 561
 562                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 563                         sess->cipher.key.data);
 564                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 565                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 566                                         sess->cipher.evp_algo,
 567                                         NULL, xform->cipher.key.data,
 568                                         NULL) != 1) {
 569                                 return -EINVAL;
 570                         }
 571                 } else if (sess->cipher.direction ==
 572                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 573                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 574                                         sess->cipher.evp_algo,
 575                                         NULL, xform->cipher.key.data,
 576                                         NULL) != 1) {
 577                                 return -EINVAL;
 578                         }
 579                 }
 580
 581                 break;
 582         default:
 583                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 584                 return -ENOTSUP;
 585         }
 586
 587         return 0;
 588 }
 589
 590 /* Set session auth parameters */
 591 static int
 592 openssl_set_session_auth_parameters(struct openssl_session *sess,
 593                 const struct rte_crypto_sym_xform *xform)
 594 {
 595         /* Select auth generate/verify */
 596         sess->auth.operation = xform->auth.op;
 597         sess->auth.algo = xform->auth.algo;
 598
 599         sess->auth.digest_length = xform->auth.digest_length;
 600
 601         /* Select auth algo */
 602         switch (xform->auth.algo) {
 603         case RTE_CRYPTO_AUTH_AES_GMAC:
 604                 /*
 605                  * OpenSSL requires GMAC to be a GCM operation
 606                  * with no cipher data length
 607                  */
 608                 sess->cipher.key.length = xform->auth.key.length;
 609
 610                 /* Set IV parameters */
 611                 sess->iv.offset = xform->auth.iv.offset;
 612                 sess->iv.length = xform->auth.iv.length;
 613
 614                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 615                         return openssl_set_sess_aead_enc_param(sess,
 616                                                 RTE_CRYPTO_AEAD_AES_GCM,
 617                                                 xform->auth.digest_length,
 618                                                 xform->auth.key.data);
 619                 else
 620                         return openssl_set_sess_aead_dec_param(sess,
 621                                                 RTE_CRYPTO_AEAD_AES_GCM,
 622                                                 xform->auth.digest_length,
 623                                                 xform->auth.key.data);
 624                 break;
 625
 626         case RTE_CRYPTO_AUTH_MD5:
 627         case RTE_CRYPTO_AUTH_SHA1:
 628         case RTE_CRYPTO_AUTH_SHA224:
 629         case RTE_CRYPTO_AUTH_SHA256:
 630         case RTE_CRYPTO_AUTH_SHA384:
 631         case RTE_CRYPTO_AUTH_SHA512:
 632                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 633                 if (get_auth_algo(xform->auth.algo,
 634                                 &sess->auth.auth.evp_algo) != 0)
 635                         return -EINVAL;
 636                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 637                 break;
 638
 639 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
 640         case RTE_CRYPTO_AUTH_MD5_HMAC:
 641         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 642         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 643         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 644         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 645         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 646                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 647
 648                 OSSL_PARAM params[2];
 649                 const char *algo;
 650                 algo = digest_name_get(xform->auth.algo);
 651                 if (!algo)
 652                         return -EINVAL;
 653                 char algo_name[MAX_OSSL_ALGO_NAME_SIZE];
 654                 rte_memcpy(algo_name, algo, (sizeof(algo)+1));
 655
 656                 EVP_MAC *mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
 657                 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac);
 658                 EVP_MAC_free(mac);
 659                 if (get_auth_algo(xform->auth.algo,
 660                                 &sess->auth.hmac.evp_algo) != 0)
 661                         return -EINVAL;
 662
 663                 params[0] = OSSL_PARAM_construct_utf8_string("digest",
 664                                         algo_name, 0);
 665                 params[1] = OSSL_PARAM_construct_end();
 666                 if (EVP_MAC_init(sess->auth.hmac.ctx,
 667                                 xform->auth.key.data,
 668                                 xform->auth.key.length,
 669                                 params) != 1)
 670                         return -EINVAL;
 671                 break;
 672 # else
 673         case RTE_CRYPTO_AUTH_MD5_HMAC:
 674         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 675         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 676         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 677         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 678         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 679                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 680                 sess->auth.hmac.ctx = HMAC_CTX_new();
 681                 if (get_auth_algo(xform->auth.algo,
 682                                 &sess->auth.hmac.evp_algo) != 0)
 683                         return -EINVAL;
 684
 685                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 686                                 xform->auth.key.data,
 687                                 xform->auth.key.length,
 688                                 sess->auth.hmac.evp_algo, NULL) != 1)
 689                         return -EINVAL;
 690                 break;
 691 # endif
 692         default:
 693                 return -ENOTSUP;
 694         }
 695
 696         return 0;
 697 }
 698
 699 /* Set session AEAD parameters */
 700 static int
 701 openssl_set_session_aead_parameters(struct openssl_session *sess,
 702                 const struct rte_crypto_sym_xform *xform)
 703 {
 704         /* Select cipher key */
 705         sess->cipher.key.length = xform->aead.key.length;
 706
 707         /* Set IV parameters */
 708         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 709                 /*
 710                  * For AES-CCM, the actual IV is placed
 711                  * one byte after the start of the IV field,
 712                  * according to the API.
 713                  */
 714                 sess->iv.offset = xform->aead.iv.offset + 1;
 715         else
 716                 sess->iv.offset = xform->aead.iv.offset;
 717
 718         sess->iv.length = xform->aead.iv.length;
 719
 720         sess->auth.aad_length = xform->aead.aad_length;
 721         sess->auth.digest_length = xform->aead.digest_length;
 722
 723         sess->aead_algo = xform->aead.algo;
 724         /* Select cipher direction */
 725         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 726                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
 727                                 xform->aead.digest_length, xform->aead.key.data);
 728         else
 729                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
 730                                 xform->aead.digest_length, xform->aead.key.data);
 731 }
 732
 733 /** Parse crypto xform chain and set private session parameters */
 734 int
 735 openssl_set_session_parameters(struct openssl_session *sess,
 736                 const struct rte_crypto_sym_xform *xform)
 737 {
 738         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 739         const struct rte_crypto_sym_xform *auth_xform = NULL;
 740         const struct rte_crypto_sym_xform *aead_xform = NULL;
 741         int ret;
 742
 743         sess->chain_order = openssl_get_chain_order(xform);
 744         switch (sess->chain_order) {
 745         case OPENSSL_CHAIN_ONLY_CIPHER:
 746                 cipher_xform = xform;
 747                 break;
 748         case OPENSSL_CHAIN_ONLY_AUTH:
 749                 auth_xform = xform;
 750                 break;
 751         case OPENSSL_CHAIN_CIPHER_AUTH:
 752                 cipher_xform = xform;
 753                 auth_xform = xform->next;
 754                 break;
 755         case OPENSSL_CHAIN_AUTH_CIPHER:
 756                 auth_xform = xform;
 757                 cipher_xform = xform->next;
 758                 break;
 759         case OPENSSL_CHAIN_COMBINED:
 760                 aead_xform = xform;
 761                 break;
 762         default:
 763                 return -EINVAL;
 764         }
 765
 766         /* Default IV length = 0 */
 767         sess->iv.length = 0;
 768
 769         /* cipher_xform must be check before auth_xform */
 770         if (cipher_xform) {
 771                 ret = openssl_set_session_cipher_parameters(
 772                                 sess, cipher_xform);
 773                 if (ret != 0) {
 774                         OPENSSL_LOG(ERR,
 775                                 "Invalid/unsupported cipher parameters");
 776                         return ret;
 777                 }
 778         }
 779
 780         if (auth_xform) {
 781                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 782                 if (ret != 0) {
 783                         OPENSSL_LOG(ERR,
 784                                 "Invalid/unsupported auth parameters");
 785                         return ret;
 786                 }
 787         }
 788
 789         if (aead_xform) {
 790                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 791                 if (ret != 0) {
 792                         OPENSSL_LOG(ERR,
 793                                 "Invalid/unsupported AEAD parameters");
 794                         return ret;
 795                 }
 796         }
 797
 798         return 0;
 799 }
 800
 801 /** Reset private session parameters */
 802 void
 803 openssl_reset_session(struct openssl_session *sess)
 804 {
 805         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 806
 807         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 808                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 809
 810         switch (sess->auth.mode) {
 811         case OPENSSL_AUTH_AS_AUTH:
 812                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 813                 break;
 814         case OPENSSL_AUTH_AS_HMAC:
 815                 EVP_PKEY_free(sess->auth.hmac.pkey);
 816 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
 817                 EVP_MAC_CTX_free(sess->auth.hmac.ctx);
 818 # else
 819                 HMAC_CTX_free(sess->auth.hmac.ctx);
 820 # endif
 821                 break;
 822         default:
 823                 break;
 824         }
 825 }
 826
 827 /** Provide session for operation */
 828 static void *
 829 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 830 {
 831         struct openssl_session *sess = NULL;
 832         struct openssl_asym_session *asym_sess = NULL;
 833
 834         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 835                 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 836                         /* get existing session */
 837                         if (likely(op->sym->session != NULL))
 838                                 sess = (struct openssl_session *)
 839                                                 get_sym_session_private_data(
 840                                                 op->sym->session,
 841                                                 cryptodev_driver_id);
 842                 } else {
 843                         if (likely(op->asym->session != NULL))
 844                                 asym_sess = (struct openssl_asym_session *)
 845                                                 op->asym->session->sess_private_data;
 846                         if (asym_sess == NULL)
 847                                 op->status =
 848                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 849                         return asym_sess;
 850                 }
 851         } else {
 852                 /* sessionless asymmetric not supported */
 853                 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
 854                         return NULL;
 855
 856                 /* provide internal session */
 857                 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
 858                 void *_sess_private_data = NULL;
 859
 860                 if (_sess == NULL)
 861                         return NULL;
 862
 863                 if (rte_mempool_get(qp->sess_mp_priv,
 864                                 (void **)&_sess_private_data))
 865                         return NULL;
 866
 867                 sess = (struct openssl_session *)_sess_private_data;
 868
 869                 if (unlikely(openssl_set_session_parameters(sess,
 870                                 op->sym->xform) != 0)) {
 871                         rte_mempool_put(qp->sess_mp, _sess);
 872                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 873                         sess = NULL;
 874                 }
 875                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 876                 set_sym_session_private_data(op->sym->session,
 877                                 cryptodev_driver_id, _sess_private_data);
 878         }
 879
 880         if (sess == NULL)
 881                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 882
 883         return sess;
 884 }
 885
 886 /*
 887  *------------------------------------------------------------------------------
 888  * Process Operations
 889  *------------------------------------------------------------------------------
 890  */
 891 static inline int
 892 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 893                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
 894 {
 895         struct rte_mbuf *m;
 896         int dstlen;
 897         int l, n = srclen;
 898         uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
 899
 900         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 901                         m = m->next)
 902                 offset -= rte_pktmbuf_data_len(m);
 903
 904         if (m == 0)
 905                 return -1;
 906
 907         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 908         if (inplace)
 909                 *dst = src;
 910
 911         l = rte_pktmbuf_data_len(m) - offset;
 912         if (srclen <= l) {
 913                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 914                         return -1;
 915                 *dst += l;
 916                 return 0;
 917         }
 918
 919         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 920                 return -1;
 921
 922         *dst += dstlen;
 923         n -= l;
 924
 925         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 926                 uint8_t diff = l - dstlen, rem;
 927
 928                 src = rte_pktmbuf_mtod(m, uint8_t *);
 929                 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
 930                 if (diff && inplace) {
 931                         rem = RTE_MIN(l,
 932                                 (EVP_CIPHER_CTX_block_size(ctx) - diff));
 933                         if (EVP_EncryptUpdate(ctx, temp,
 934                                                 &dstlen, src, rem) <= 0)
 935                                 return -1;
 936                         n -= rem;
 937                         rte_memcpy(*dst, temp, diff);
 938                         rte_memcpy(src, temp + diff, rem);
 939                         src += rem;
 940                         l -= rem;
 941                 }
 942                 if (inplace)
 943                         *dst = src;
 944                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 945                         return -1;
 946                 *dst += dstlen;
 947                 n -= l;
 948         }
 949
 950         return 0;
 951 }
 952
 953 static inline int
 954 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 955                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
 956 {
 957         struct rte_mbuf *m;
 958         int dstlen;
 959         int l, n = srclen;
 960         uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
 961
 962         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 963                         m = m->next)
 964                 offset -= rte_pktmbuf_data_len(m);
 965
 966         if (m == 0)
 967                 return -1;
 968
 969         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 970         if (inplace)
 971                 *dst = src;
 972
 973         l = rte_pktmbuf_data_len(m) - offset;
 974         if (srclen <= l) {
 975                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 976                         return -1;
 977                 *dst += l;
 978                 return 0;
 979         }
 980
 981         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 982                 return -1;
 983
 984         *dst += dstlen;
 985         n -= l;
 986
 987         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 988                 uint8_t diff = l - dstlen, rem;
 989
 990                 src = rte_pktmbuf_mtod(m, uint8_t *);
 991                 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
 992                 if (diff && inplace) {
 993                         rem = RTE_MIN(l,
 994                                 (EVP_CIPHER_CTX_block_size(ctx) - diff));
 995                         if (EVP_DecryptUpdate(ctx, temp,
 996                                                 &dstlen, src, rem) <= 0)
 997                                 return -1;
 998                         n -= rem;
 999                         rte_memcpy(*dst, temp, diff);
1000                         rte_memcpy(src, temp + diff, rem);
1001                         src += rem;
1002                         l -= rem;
1003                 }
1004                 if (inplace)
1005                         *dst = src;
1006                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
1007                         return -1;
1008                 *dst += dstlen;
1009                 n -= l;
1010         }
1011
1012         return 0;
1013 }
1014
1015 /** Process standard openssl cipher encryption */
1016 static int
1017 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1018                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1019                 uint8_t inplace)
1020 {
1021         int totlen;
1022
1023         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1024                 goto process_cipher_encrypt_err;
1025
1026         EVP_CIPHER_CTX_set_padding(ctx, 0);
1027
1028         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1029                         srclen, ctx, inplace))
1030                 goto process_cipher_encrypt_err;
1031
1032         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
1033                 goto process_cipher_encrypt_err;
1034
1035         return 0;
1036
1037 process_cipher_encrypt_err:
1038         OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed");
1039         return -EINVAL;
1040 }
1041
1042 /** Process standard openssl cipher encryption */
1043 static int
1044 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
1045                 uint8_t *iv, int srclen,
1046                 EVP_CIPHER_CTX *ctx)
1047 {
1048         uint8_t i;
1049         uint8_t encrypted_iv[DES_BLOCK_SIZE];
1050         int encrypted_ivlen;
1051
1052         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
1053                         iv, DES_BLOCK_SIZE) <= 0)
1054                 goto process_cipher_encrypt_err;
1055
1056         for (i = 0; i < srclen; i++)
1057                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
1058
1059         return 0;
1060
1061 process_cipher_encrypt_err:
1062         OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed");
1063         return -EINVAL;
1064 }
1065 /** Process standard openssl cipher decryption */
1066 static int
1067 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1068                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1069                 uint8_t inplace)
1070 {
1071         int totlen;
1072
1073         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1074                 goto process_cipher_decrypt_err;
1075
1076         EVP_CIPHER_CTX_set_padding(ctx, 0);
1077
1078         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1079                         srclen, ctx, inplace))
1080                 goto process_cipher_decrypt_err;
1081
1082         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
1083                 goto process_cipher_decrypt_err;
1084         return 0;
1085
1086 process_cipher_decrypt_err:
1087         OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed");
1088         return -EINVAL;
1089 }
1090
1091 /** Process cipher des 3 ctr encryption, decryption algorithm */
1092 static int
1093 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
1094                 int offset, uint8_t *iv, uint8_t *key, int srclen,
1095                 EVP_CIPHER_CTX *ctx)
1096 {
1097         uint8_t ebuf[8], ctr[8];
1098         int unused, n;
1099         struct rte_mbuf *m;
1100         uint8_t *src;
1101         int l;
1102
1103         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1104                         m = m->next)
1105                 offset -= rte_pktmbuf_data_len(m);
1106
1107         if (m == 0)
1108                 goto process_cipher_des3ctr_err;
1109
1110         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1111         l = rte_pktmbuf_data_len(m) - offset;
1112
1113         /* We use 3DES encryption also for decryption.
1114          * IV is not important for 3DES ecb
1115          */
1116         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
1117                 goto process_cipher_des3ctr_err;
1118
1119         memcpy(ctr, iv, 8);
1120
1121         for (n = 0; n < srclen; n++) {
1122                 if (n % 8 == 0) {
1123                         if (EVP_EncryptUpdate(ctx,
1124                                         (unsigned char *)&ebuf, &unused,
1125                                         (const unsigned char *)&ctr, 8) <= 0)
1126                                 goto process_cipher_des3ctr_err;
1127                         ctr_inc(ctr);
1128                 }
1129                 dst[n] = *(src++) ^ ebuf[n % 8];
1130
1131                 l--;
1132                 if (!l) {
1133                         m = m->next;
1134                         if (m) {
1135                                 src = rte_pktmbuf_mtod(m, uint8_t *);
1136                                 l = rte_pktmbuf_data_len(m);
1137                         }
1138                 }
1139         }
1140
1141         return 0;
1142
1143 process_cipher_des3ctr_err:
1144         OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed");
1145         return -EINVAL;
1146 }
1147
1148 /** Process AES-GCM encrypt algorithm */
1149 static int
1150 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1151                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1152                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1153 {
1154         int len = 0, unused = 0;
1155         uint8_t empty[] = {};
1156
1157         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1158                 goto process_auth_encryption_gcm_err;
1159
1160         if (aadlen > 0)
1161                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1162                         goto process_auth_encryption_gcm_err;
1163
1164         if (srclen > 0)
1165                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1166                                 srclen, ctx, 0))
1167                         goto process_auth_encryption_gcm_err;
1168
1169         /* Workaround open ssl bug in version less then 1.0.1f */
1170         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1171                 goto process_auth_encryption_gcm_err;
1172
1173         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1174                 goto process_auth_encryption_gcm_err;
1175
1176         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1177                 goto process_auth_encryption_gcm_err;
1178
1179         return 0;
1180
1181 process_auth_encryption_gcm_err:
1182         OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed");
1183         return -EINVAL;
1184 }
1185
1186 /** Process AES-CCM encrypt algorithm */
1187 static int
1188 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1189                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1190                 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1191 {
1192         int len = 0;
1193
1194         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1195                 goto process_auth_encryption_ccm_err;
1196
1197         if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1198                 goto process_auth_encryption_ccm_err;
1199
1200         if (aadlen > 0)
1201                 /*
1202                  * For AES-CCM, the actual AAD is placed
1203                  * 18 bytes after the start of the AAD field,
1204                  * according to the API.
1205                  */
1206                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1207                         goto process_auth_encryption_ccm_err;
1208
1209         if (srclen >= 0)
1210                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1211                                 srclen, ctx, 0))
1212                         goto process_auth_encryption_ccm_err;
1213
1214         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1215                 goto process_auth_encryption_ccm_err;
1216
1217         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1218                 goto process_auth_encryption_ccm_err;
1219
1220         return 0;
1221
1222 process_auth_encryption_ccm_err:
1223         OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed");
1224         return -EINVAL;
1225 }
1226
1227 /** Process AES-GCM decrypt algorithm */
1228 static int
1229 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1230                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1231                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1232 {
1233         int len = 0, unused = 0;
1234         uint8_t empty[] = {};
1235
1236         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1237                 goto process_auth_decryption_gcm_err;
1238
1239         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1240                 goto process_auth_decryption_gcm_err;
1241
1242         if (aadlen > 0)
1243                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1244                         goto process_auth_decryption_gcm_err;
1245
1246         if (srclen > 0)
1247                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1248                                 srclen, ctx, 0))
1249                         goto process_auth_decryption_gcm_err;
1250
1251         /* Workaround open ssl bug in version less then 1.0.1f */
1252         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1253                 goto process_auth_decryption_gcm_err;
1254
1255         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1256                 return -EFAULT;
1257
1258         return 0;
1259
1260 process_auth_decryption_gcm_err:
1261         OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed");
1262         return -EINVAL;
1263 }
1264
1265 /** Process AES-CCM decrypt algorithm */
1266 static int
1267 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1268                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1269                 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1270                 EVP_CIPHER_CTX *ctx)
1271 {
1272         int len = 0;
1273
1274         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1275                 goto process_auth_decryption_ccm_err;
1276
1277         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1278                 goto process_auth_decryption_ccm_err;
1279
1280         if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1281                 goto process_auth_decryption_ccm_err;
1282
1283         if (aadlen > 0)
1284                 /*
1285                  * For AES-CCM, the actual AAD is placed
1286                  * 18 bytes after the start of the AAD field,
1287                  * according to the API.
1288                  */
1289                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1290                         goto process_auth_decryption_ccm_err;
1291
1292         if (srclen >= 0)
1293                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1294                                 srclen, ctx, 0))
1295                         return -EFAULT;
1296
1297         return 0;
1298
1299 process_auth_decryption_ccm_err:
1300         OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed");
1301         return -EINVAL;
1302 }
1303
1304 /** Process standard openssl auth algorithms */
1305 static int
1306 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1307                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1308                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1309 {
1310         size_t dstlen;
1311         struct rte_mbuf *m;
1312         int l, n = srclen;
1313         uint8_t *src;
1314
1315         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1316                         m = m->next)
1317                 offset -= rte_pktmbuf_data_len(m);
1318
1319         if (m == 0)
1320                 goto process_auth_err;
1321
1322         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1323                 goto process_auth_err;
1324
1325         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1326
1327         l = rte_pktmbuf_data_len(m) - offset;
1328         if (srclen <= l) {
1329                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1330                         goto process_auth_err;
1331                 goto process_auth_final;
1332         }
1333
1334         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1335                 goto process_auth_err;
1336
1337         n -= l;
1338
1339         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1340                 src = rte_pktmbuf_mtod(m, uint8_t *);
1341                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1342                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1343                         goto process_auth_err;
1344                 n -= l;
1345         }
1346
1347 process_auth_final:
1348         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1349                 goto process_auth_err;
1350         return 0;
1351
1352 process_auth_err:
1353         OPENSSL_LOG(ERR, "Process openssl auth failed");
1354         return -EINVAL;
1355 }
1356
1357 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1358 /** Process standard openssl auth algorithms with hmac */
1359 static int
1360 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1361                 int srclen, EVP_MAC_CTX *ctx)
1362 {
1363         size_t dstlen;
1364         struct rte_mbuf *m;
1365         int l, n = srclen;
1366         uint8_t *src;
1367
1368         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1369                         m = m->next)
1370                 offset -= rte_pktmbuf_data_len(m);
1371
1372         if (m == 0)
1373                 goto process_auth_err;
1374
1375         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1376
1377         l = rte_pktmbuf_data_len(m) - offset;
1378         if (srclen <= l) {
1379                 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1)
1380                         goto process_auth_err;
1381                 goto process_auth_final;
1382         }
1383
1384         if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1385                 goto process_auth_err;
1386
1387         n -= l;
1388
1389         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1390                 src = rte_pktmbuf_mtod(m, uint8_t *);
1391                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1392                 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1393                         goto process_auth_err;
1394                 n -= l;
1395         }
1396
1397 process_auth_final:
1398         if (EVP_MAC_final(ctx, dst, &dstlen, sizeof(dst)) != 1)
1399                 goto process_auth_err;
1400
1401         EVP_MAC_CTX_free(ctx);
1402         return 0;
1403
1404 process_auth_err:
1405         EVP_MAC_CTX_free(ctx);
1406         OPENSSL_LOG(ERR, "Process openssl auth failed");
1407         return -EINVAL;
1408 }
1409 # else
1410 /** Process standard openssl auth algorithms with hmac */
1411 static int
1412 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1413                 int srclen, HMAC_CTX *ctx)
1414 {
1415         unsigned int dstlen;
1416         struct rte_mbuf *m;
1417         int l, n = srclen;
1418         uint8_t *src;
1419
1420         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1421                         m = m->next)
1422                 offset -= rte_pktmbuf_data_len(m);
1423
1424         if (m == 0)
1425                 goto process_auth_err;
1426
1427         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1428
1429         l = rte_pktmbuf_data_len(m) - offset;
1430         if (srclen <= l) {
1431                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1432                         goto process_auth_err;
1433                 goto process_auth_final;
1434         }
1435
1436         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1437                 goto process_auth_err;
1438
1439         n -= l;
1440
1441         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1442                 src = rte_pktmbuf_mtod(m, uint8_t *);
1443                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1444                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1445                         goto process_auth_err;
1446                 n -= l;
1447         }
1448
1449 process_auth_final:
1450         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1451                 goto process_auth_err;
1452
1453         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1454                 goto process_auth_err;
1455
1456         return 0;
1457
1458 process_auth_err:
1459         OPENSSL_LOG(ERR, "Process openssl auth failed");
1460         return -EINVAL;
1461 }
1462 # endif
1463 /*----------------------------------------------------------------------------*/
1464
1465 /** Process auth/cipher combined operation */
1466 static void
1467 process_openssl_combined_op
1468                 (struct rte_crypto_op *op, struct openssl_session *sess,
1469                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1470 {
1471         /* cipher */
1472         uint8_t *dst = NULL, *iv, *tag, *aad;
1473         int srclen, aadlen, status = -1;
1474         uint32_t offset;
1475         uint8_t taglen;
1476
1477         /*
1478          * Segmented destination buffer is not supported for
1479          * encryption/decryption
1480          */
1481         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1482                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1483                 return;
1484         }
1485
1486         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1487                         sess->iv.offset);
1488         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1489                 srclen = 0;
1490                 offset = op->sym->auth.data.offset;
1491                 aadlen = op->sym->auth.data.length;
1492                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1493                                 op->sym->auth.data.offset);
1494                 tag = op->sym->auth.digest.data;
1495                 if (tag == NULL)
1496                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1497                                 offset + aadlen);
1498         } else {
1499                 srclen = op->sym->aead.data.length;
1500                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1501                                 op->sym->aead.data.offset);
1502                 offset = op->sym->aead.data.offset;
1503                 aad = op->sym->aead.aad.data;
1504                 aadlen = sess->auth.aad_length;
1505                 tag = op->sym->aead.digest.data;
1506                 if (tag == NULL)
1507                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1508                                 offset + srclen);
1509         }
1510
1511         taglen = sess->auth.digest_length;
1512
1513         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1514                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1515                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1516                         status = process_openssl_auth_encryption_gcm(
1517                                         mbuf_src, offset, srclen,
1518                                         aad, aadlen, iv,
1519                                         dst, tag, sess->cipher.ctx);
1520                 else
1521                         status = process_openssl_auth_encryption_ccm(
1522                                         mbuf_src, offset, srclen,
1523                                         aad, aadlen, iv,
1524                                         dst, tag, taglen, sess->cipher.ctx);
1525
1526         } else {
1527                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1528                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1529                         status = process_openssl_auth_decryption_gcm(
1530                                         mbuf_src, offset, srclen,
1531                                         aad, aadlen, iv,
1532                                         dst, tag, sess->cipher.ctx);
1533                 else
1534                         status = process_openssl_auth_decryption_ccm(
1535                                         mbuf_src, offset, srclen,
1536                                         aad, aadlen, iv,
1537                                         dst, tag, taglen, sess->cipher.ctx);
1538         }
1539
1540         if (status != 0) {
1541                 if (status == (-EFAULT) &&
1542                                 sess->auth.operation ==
1543                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1544                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1545                 else
1546                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1547         }
1548 }
1549
1550 /** Process cipher operation */
1551 static void
1552 process_openssl_cipher_op
1553                 (struct rte_crypto_op *op, struct openssl_session *sess,
1554                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1555 {
1556         uint8_t *dst, *iv;
1557         int srclen, status;
1558         uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0;
1559         EVP_CIPHER_CTX *ctx_copy;
1560
1561         /*
1562          * Segmented OOP destination buffer is not supported for encryption/
1563          * decryption. In case of des3ctr, even inplace segmented buffers are
1564          * not supported.
1565          */
1566         if (!rte_pktmbuf_is_contiguous(mbuf_dst) &&
1567                         (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) {
1568                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1569                 return;
1570         }
1571
1572         srclen = op->sym->cipher.data.length;
1573         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1574                         op->sym->cipher.data.offset);
1575
1576         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1577                         sess->iv.offset);
1578         ctx_copy = EVP_CIPHER_CTX_new();
1579         EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx);
1580
1581         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1582                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1583                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1584                                         op->sym->cipher.data.offset, iv,
1585                                         srclen, ctx_copy, inplace);
1586                 else
1587                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1588                                         op->sym->cipher.data.offset, iv,
1589                                         srclen, ctx_copy, inplace);
1590         else
1591                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1592                                 op->sym->cipher.data.offset, iv,
1593                                 sess->cipher.key.data, srclen,
1594                                 ctx_copy);
1595
1596         EVP_CIPHER_CTX_free(ctx_copy);
1597         if (status != 0)
1598                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1599 }
1600
1601 /** Process cipher operation */
1602 static void
1603 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1604                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1605                 struct rte_mbuf *mbuf_dst)
1606 {
1607         uint8_t *src, *dst, *iv;
1608         uint8_t block_size, last_block_len;
1609         int srclen, status = 0;
1610
1611         srclen = op->sym->cipher.data.length;
1612         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1613                         op->sym->cipher.data.offset);
1614         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1615                         op->sym->cipher.data.offset);
1616
1617         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1618                         sess->iv.offset);
1619
1620         block_size = DES_BLOCK_SIZE;
1621
1622         last_block_len = srclen % block_size;
1623         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1624                 /* Encrypt only with ECB mode XOR IV */
1625                 if (srclen < block_size) {
1626                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1627                                         iv, srclen,
1628                                         sess->cipher.bpi_ctx);
1629                 } else {
1630                         srclen -= last_block_len;
1631                         /* Encrypt with the block aligned stream with CBC mode */
1632                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1633                                         op->sym->cipher.data.offset, iv,
1634                                         srclen, sess->cipher.ctx, 0);
1635                         if (last_block_len) {
1636                                 /* Point at last block */
1637                                 dst += srclen;
1638                                 /*
1639                                  * IV is the last encrypted block from
1640                                  * the previous operation
1641                                  */
1642                                 iv = dst - block_size;
1643                                 src += srclen;
1644                                 srclen = last_block_len;
1645                                 /* Encrypt the last frame with ECB mode */
1646                                 status |= process_openssl_cipher_bpi_encrypt(src,
1647                                                 dst, iv,
1648                                                 srclen, sess->cipher.bpi_ctx);
1649                         }
1650                 }
1651         } else {
1652                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1653                 if (srclen < block_size) {
1654                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1655                                         iv,
1656                                         srclen,
1657                                         sess->cipher.bpi_ctx);
1658                 } else {
1659                         if (last_block_len) {
1660                                 /* Point at last block */
1661                                 dst += srclen - last_block_len;
1662                                 src += srclen - last_block_len;
1663                                 /*
1664                                  * IV is the last full block
1665                                  */
1666                                 iv = src - block_size;
1667                                 /*
1668                                  * Decrypt the last frame with ECB mode
1669                                  * (encrypt, as it is the same operation)
1670                                  */
1671                                 status = process_openssl_cipher_bpi_encrypt(src,
1672                                                 dst, iv,
1673                                                 last_block_len, sess->cipher.bpi_ctx);
1674                                 /* Prepare parameters for CBC mode op */
1675                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1676                                                 sess->iv.offset);
1677                                 dst += last_block_len - srclen;
1678                                 srclen -= last_block_len;
1679                         }
1680
1681                         /* Decrypt with CBC mode */
1682                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1683                                         op->sym->cipher.data.offset, iv,
1684                                         srclen, sess->cipher.ctx, 0);
1685                 }
1686         }
1687
1688         if (status != 0)
1689                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1690 }
1691
1692 /** Process auth operation */
1693 static void
1694 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1695                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1696                 struct rte_mbuf *mbuf_dst)
1697 {
1698         uint8_t *dst;
1699         int srclen, status;
1700         EVP_MD_CTX *ctx_a;
1701 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1702         EVP_MAC_CTX *ctx_h;
1703         EVP_MAC *mac;
1704 # else
1705         HMAC_CTX *ctx_h;
1706 # endif
1707
1708         srclen = op->sym->auth.data.length;
1709
1710         dst = qp->temp_digest;
1711
1712         switch (sess->auth.mode) {
1713         case OPENSSL_AUTH_AS_AUTH:
1714                 ctx_a = EVP_MD_CTX_create();
1715                 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx);
1716                 status = process_openssl_auth(mbuf_src, dst,
1717                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1718                                 ctx_a, sess->auth.auth.evp_algo);
1719                 EVP_MD_CTX_destroy(ctx_a);
1720                 break;
1721         case OPENSSL_AUTH_AS_HMAC:
1722 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1723                 mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
1724                 ctx_h = EVP_MAC_CTX_new(mac);
1725                 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx);
1726                 EVP_MAC_free(mac);
1727                 status = process_openssl_auth_hmac(mbuf_src, dst,
1728                                 op->sym->auth.data.offset, srclen,
1729                                 ctx_h);
1730 # else
1731                 ctx_h = HMAC_CTX_new();
1732                 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx);
1733                 status = process_openssl_auth_hmac(mbuf_src, dst,
1734                                 op->sym->auth.data.offset, srclen,
1735                                 ctx_h);
1736                 HMAC_CTX_free(ctx_h);
1737 # endif
1738                 break;
1739         default:
1740                 status = -1;
1741                 break;
1742         }
1743
1744         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1745                 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data,
1746                                 sess->auth.digest_length) != 0) {
1747                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1748                 }
1749         } else {
1750                 uint8_t *auth_dst;
1751
1752                 auth_dst = op->sym->auth.digest.data;
1753                 if (auth_dst == NULL)
1754                         auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1755                                         op->sym->auth.data.offset +
1756                                         op->sym->auth.data.length);
1757                 memcpy(auth_dst, dst, sess->auth.digest_length);
1758         }
1759
1760         if (status != 0)
1761                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1762 }
1763
1764 /* process dsa sign operation */
1765 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
1766 static int
1767 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop,
1768                 struct openssl_asym_session *sess)
1769 {
1770         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1771         EVP_PKEY_CTX *dsa_ctx = NULL;
1772         EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL);
1773         EVP_PKEY *pkey = NULL;
1774         OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld;
1775         OSSL_PARAM *params = NULL;
1776
1777         size_t outlen;
1778         unsigned char *dsa_sign_data;
1779         const unsigned char *dsa_sign_data_p;
1780
1781         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1782         params = OSSL_PARAM_BLD_to_param(param_bld);
1783         if (!params) {
1784                 OSSL_PARAM_BLD_free(param_bld);
1785                 return -1;
1786         }
1787
1788         if (key_ctx == NULL
1789                 || EVP_PKEY_fromdata_init(key_ctx) <= 0
1790                 || EVP_PKEY_fromdata(key_ctx, &pkey,
1791                                                 EVP_PKEY_PUBLIC_KEY, params) <= 0)
1792                 goto err_dsa_sign;
1793
1794         dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL);
1795         if (!dsa_ctx)
1796                 goto err_dsa_sign;
1797
1798         if (EVP_PKEY_sign_init(dsa_ctx) <= 0)
1799                 goto err_dsa_sign;
1800
1801         if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data,
1802                                                 op->message.length) <= 0)
1803                 goto err_dsa_sign;
1804
1805         if (outlen <= 0)
1806                 goto err_dsa_sign;
1807
1808         dsa_sign_data = OPENSSL_malloc(outlen);
1809         if (!dsa_sign_data)
1810                 goto err_dsa_sign;
1811
1812         if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data,
1813                                                 op->message.length) <= 0) {
1814                 free(dsa_sign_data);
1815                 goto err_dsa_sign;
1816         }
1817
1818         dsa_sign_data_p = (const unsigned char *)dsa_sign_data;
1819         DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen);
1820         if (!sign) {
1821                 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
1822                 free(dsa_sign_data);
1823                 goto err_dsa_sign;
1824         } else {
1825                 const BIGNUM *r = NULL, *s = NULL;
1826                 get_dsa_sign(sign, &r, &s);
1827
1828                 op->r.length = BN_bn2bin(r, op->r.data);
1829                 op->s.length = BN_bn2bin(s, op->s.data);
1830                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1831         }
1832
1833         DSA_SIG_free(sign);
1834         free(dsa_sign_data);
1835         return 0;
1836
1837 err_dsa_sign:
1838         if (params)
1839                 OSSL_PARAM_free(params);
1840         if (key_ctx)
1841                 EVP_PKEY_CTX_free(key_ctx);
1842         if (dsa_ctx)
1843                 EVP_PKEY_CTX_free(dsa_ctx);
1844         return -1;
1845 }
1846
1847 /* process dsa verify operation */
1848 static int
1849 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop,
1850                 struct openssl_asym_session *sess)
1851 {
1852         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1853         DSA_SIG *sign = DSA_SIG_new();
1854         BIGNUM *r = NULL, *s = NULL;
1855         BIGNUM *pub_key = NULL;
1856         OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld;
1857         OSSL_PARAM *params = NULL;
1858         EVP_PKEY *pkey = NULL;
1859         EVP_PKEY_CTX *dsa_ctx = NULL;
1860         EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL);
1861         unsigned char *dsa_sig = NULL;
1862         size_t sig_len;
1863         int ret = -1;
1864
1865         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1866         if (!param_bld) {
1867                 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
1868                 return -1;
1869         }
1870
1871         r = BN_bin2bn(op->r.data, op->r.length, r);
1872         s = BN_bin2bn(op->s.data, op->s.length, s);
1873         pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key);
1874         if (!r || !s || !pub_key) {
1875                 BN_free(r);
1876                 BN_free(s);
1877                 BN_free(pub_key);
1878                 OSSL_PARAM_BLD_free(param_bld);
1879                 goto err_dsa_verify;
1880         }
1881
1882         set_dsa_sign(sign, r, s);
1883         if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) {
1884                 OSSL_PARAM_BLD_free(param_bld);
1885                 goto err_dsa_verify;
1886         }
1887
1888         params = OSSL_PARAM_BLD_to_param(param_bld);
1889         if (!params) {
1890                 OSSL_PARAM_BLD_free(param_bld);
1891                 goto err_dsa_verify;
1892         }
1893
1894         if (key_ctx == NULL
1895                 || EVP_PKEY_fromdata_init(key_ctx) <= 0
1896                 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0)
1897                 goto err_dsa_verify;
1898
1899         dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL);
1900         if (!dsa_ctx)
1901                 goto err_dsa_verify;
1902
1903         if (!sign)
1904                 goto err_dsa_verify;
1905
1906         sig_len = i2d_DSA_SIG(sign, &dsa_sig);
1907         if (EVP_PKEY_verify_init(dsa_ctx) <= 0)
1908                 goto err_dsa_verify;
1909
1910         ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len,
1911                                         op->message.data, op->message.length);
1912         if (ret == 1) {
1913                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1914                 ret = 0;
1915         }
1916
1917 err_dsa_verify:
1918         if (sign)
1919                 DSA_SIG_free(sign);
1920         if (params)
1921                 OSSL_PARAM_free(params);
1922         if (key_ctx)
1923                 EVP_PKEY_CTX_free(key_ctx);
1924         if (dsa_ctx)
1925                 EVP_PKEY_CTX_free(dsa_ctx);
1926
1927         return ret;
1928 }
1929 #else
1930 static int
1931 process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
1932                 struct openssl_asym_session *sess)
1933 {
1934         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1935         DSA *dsa = sess->u.s.dsa;
1936         DSA_SIG *sign = NULL;
1937
1938         sign = DSA_do_sign(op->message.data,
1939                         op->message.length,
1940                         dsa);
1941
1942         if (sign == NULL) {
1943                 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
1944                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1945         } else {
1946                 const BIGNUM *r = NULL, *s = NULL;
1947                 get_dsa_sign(sign, &r, &s);
1948
1949                 op->r.length = BN_bn2bin(r, op->r.data);
1950                 op->s.length = BN_bn2bin(s, op->s.data);
1951                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1952         }
1953
1954         DSA_SIG_free(sign);
1955
1956         return 0;
1957 }
1958
1959 /* process dsa verify operation */
1960 static int
1961 process_openssl_dsa_verify_op(struct rte_crypto_op *cop,
1962                 struct openssl_asym_session *sess)
1963 {
1964         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1965         DSA *dsa = sess->u.s.dsa;
1966         int ret;
1967         DSA_SIG *sign = DSA_SIG_new();
1968         BIGNUM *r = NULL, *s = NULL;
1969         BIGNUM *pub_key = NULL;
1970
1971         if (sign == NULL) {
1972                 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
1973                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1974                 return -1;
1975         }
1976
1977         r = BN_bin2bn(op->r.data,
1978                         op->r.length,
1979                         r);
1980         s = BN_bin2bn(op->s.data,
1981                         op->s.length,
1982                         s);
1983         pub_key = BN_bin2bn(op->y.data,
1984                         op->y.length,
1985                         pub_key);
1986         if (!r || !s || !pub_key) {
1987                 BN_free(r);
1988                 BN_free(s);
1989                 BN_free(pub_key);
1990
1991                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1992                 return -1;
1993         }
1994         set_dsa_sign(sign, r, s);
1995         set_dsa_pub_key(dsa, pub_key);
1996
1997         ret = DSA_do_verify(op->message.data,
1998                         op->message.length,
1999                         sign,
2000                         dsa);
2001
2002         if (ret != 1)
2003                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2004         else
2005                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2006
2007         DSA_SIG_free(sign);
2008
2009         return 0;
2010 }
2011 #endif
2012
2013 /* process dh operation */
2014 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2015 static int
2016 process_openssl_dh_op_evp(struct rte_crypto_op *cop,
2017                 struct openssl_asym_session *sess)
2018 {
2019         struct rte_crypto_dh_op_param *op = &cop->asym->dh;
2020         OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld;
2021         OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer;
2022         OSSL_PARAM *params = NULL;
2023         EVP_PKEY *dhpkey = NULL;
2024         EVP_PKEY *peerkey = NULL;
2025         BIGNUM *priv_key = NULL;
2026         BIGNUM *pub_key = NULL;
2027         int ret = -1;
2028
2029         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2030         EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
2031         if (dh_ctx == NULL || param_bld == NULL)
2032                 return ret;
2033
2034         if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2035                 OSSL_PARAM *params_peer = NULL;
2036
2037                 if (!param_bld_peer)
2038                         return ret;
2039
2040                 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length,
2041                                         pub_key);
2042                 if (pub_key == NULL) {
2043                         OSSL_PARAM_BLD_free(param_bld_peer);
2044                         return ret;
2045                 }
2046
2047                 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY,
2048                                 pub_key)) {
2049                         OPENSSL_LOG(ERR, "Failed to set public key\n");
2050                         OSSL_PARAM_BLD_free(param_bld_peer);
2051                         BN_free(pub_key);
2052                         return ret;
2053                 }
2054
2055                 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer);
2056                 if (!params_peer) {
2057                         OSSL_PARAM_BLD_free(param_bld_peer);
2058                         BN_free(pub_key);
2059                         return ret;
2060                 }
2061
2062                 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
2063                 if (EVP_PKEY_keygen_init(peer_ctx) != 1) {
2064                         OSSL_PARAM_free(params_peer);
2065                         BN_free(pub_key);
2066                         return ret;
2067                 }
2068
2069                 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) {
2070                         EVP_PKEY_CTX_free(peer_ctx);
2071                         OSSL_PARAM_free(params_peer);
2072                         BN_free(pub_key);
2073                         return ret;
2074                 }
2075
2076                 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) {
2077                         EVP_PKEY_CTX_free(peer_ctx);
2078                         OSSL_PARAM_free(params_peer);
2079                         BN_free(pub_key);
2080                         return ret;
2081                 }
2082
2083                 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length,
2084                                         priv_key);
2085                 if (priv_key == NULL) {
2086                         EVP_PKEY_CTX_free(peer_ctx);
2087                         OSSL_PARAM_free(params_peer);
2088                         BN_free(pub_key);
2089                         return ret;
2090                 }
2091
2092                 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY,
2093                                 priv_key)) {
2094                         OPENSSL_LOG(ERR, "Failed to set private key\n");
2095                         EVP_PKEY_CTX_free(peer_ctx);
2096                         OSSL_PARAM_free(params_peer);
2097                         BN_free(pub_key);
2098                         BN_free(priv_key);
2099                         return ret;
2100                 }
2101
2102                 OSSL_PARAM_free(params_peer);
2103                 EVP_PKEY_CTX_free(peer_ctx);
2104         }
2105
2106         params = OSSL_PARAM_BLD_to_param(param_bld);
2107         if (!params)
2108                 goto err_dh;
2109
2110         if (EVP_PKEY_keygen_init(dh_ctx) != 1)
2111                 goto err_dh;
2112
2113         if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1)
2114                 goto err_dh;
2115
2116         if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1)
2117                 goto err_dh;
2118
2119         if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
2120                 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__);
2121                 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key))
2122                         goto err_dh;
2123                                 /* output public key */
2124                 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data);
2125         }
2126
2127         if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
2128
2129                 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__);
2130                 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key))
2131                         goto err_dh;
2132
2133                 /* provide generated private key back to user */
2134                 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data);
2135         }
2136
2137         if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2138                 size_t skey_len;
2139                 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL);
2140                 if (!sc_ctx)
2141                         goto err_dh;
2142
2143                 if (EVP_PKEY_derive_init(sc_ctx) <= 0) {
2144                         EVP_PKEY_CTX_free(sc_ctx);
2145                         goto err_dh;
2146                 }
2147
2148                 if (!peerkey) {
2149                         EVP_PKEY_CTX_free(sc_ctx);
2150                         goto err_dh;
2151                 }
2152
2153                 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) {
2154                         EVP_PKEY_CTX_free(sc_ctx);
2155                         goto err_dh;
2156                 }
2157
2158                 /* Determine buffer length */
2159                 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) {
2160                         EVP_PKEY_CTX_free(sc_ctx);
2161                         goto err_dh;
2162                 }
2163
2164                 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) {
2165                         EVP_PKEY_CTX_free(sc_ctx);
2166                         goto err_dh;
2167                 }
2168
2169                 op->shared_secret.length = skey_len;
2170                 EVP_PKEY_CTX_free(sc_ctx);
2171         }
2172
2173         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2174         ret = 0;
2175
2176  err_dh:
2177         if (pub_key)
2178                 BN_free(pub_key);
2179         if (priv_key)
2180                 BN_free(priv_key);
2181         if (params)
2182                 OSSL_PARAM_free(params);
2183         if (dhpkey)
2184                 EVP_PKEY_free(dhpkey);
2185         if (peerkey)
2186                 EVP_PKEY_free(peerkey);
2187
2188         EVP_PKEY_CTX_free(dh_ctx);
2189
2190         return ret;
2191 }
2192 #else
2193 static int
2194 process_openssl_dh_op(struct rte_crypto_op *cop,
2195                 struct openssl_asym_session *sess)
2196 {
2197         struct rte_crypto_dh_op_param *op = &cop->asym->dh;
2198         struct rte_crypto_asym_op *asym_op = cop->asym;
2199         DH *dh_key = sess->u.dh.dh_key;
2200         BIGNUM *priv_key = NULL;
2201         int ret = 0;
2202
2203         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2204                 /* compute shared secret using peer public key
2205                  * and current private key
2206                  * shared secret = peer_key ^ priv_key mod p
2207                  */
2208                 BIGNUM *peer_key = NULL;
2209
2210                 /* copy private key and peer key and compute shared secret */
2211                 peer_key = BN_bin2bn(op->pub_key.data,
2212                                 op->pub_key.length,
2213                                 peer_key);
2214                 if (peer_key == NULL) {
2215                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2216                         return -1;
2217                 }
2218                 priv_key = BN_bin2bn(op->priv_key.data,
2219                                 op->priv_key.length,
2220                                 priv_key);
2221                 if (priv_key == NULL) {
2222                         BN_free(peer_key);
2223                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2224                         return -1;
2225                 }
2226                 ret = set_dh_priv_key(dh_key, priv_key);
2227                 if (ret) {
2228                         OPENSSL_LOG(ERR, "Failed to set private key\n");
2229                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2230                         BN_free(peer_key);
2231                         BN_free(priv_key);
2232                         return 0;
2233                 }
2234
2235                 ret = DH_compute_key(
2236                                 op->shared_secret.data,
2237                                 peer_key, dh_key);
2238                 if (ret < 0) {
2239                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2240                         BN_free(peer_key);
2241                         /* priv key is already loaded into dh,
2242                          * let's not free that directly here.
2243                          * DH_free() will auto free it later.
2244                          */
2245                         return 0;
2246                 }
2247                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2248                 op->shared_secret.length = ret;
2249                 BN_free(peer_key);
2250                 return 0;
2251         }
2252
2253         /*
2254          * other options are public and private key generations.
2255          *
2256          * if user provides private key,
2257          * then first set DH with user provided private key
2258          */
2259         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE &&
2260                         op->priv_key.length) {
2261                 /* generate public key using user-provided private key
2262                  * pub_key = g ^ priv_key mod p
2263                  */
2264
2265                 /* load private key into DH */
2266                 priv_key = BN_bin2bn(op->priv_key.data,
2267                                 op->priv_key.length,
2268                                 priv_key);
2269                 if (priv_key == NULL) {
2270                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2271                         return -1;
2272                 }
2273                 ret = set_dh_priv_key(dh_key, priv_key);
2274                 if (ret) {
2275                         OPENSSL_LOG(ERR, "Failed to set private key\n");
2276                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2277                         BN_free(priv_key);
2278                         return 0;
2279                 }
2280         }
2281
2282         /* generate public and private key pair.
2283          *
2284          * if private key already set, generates only public key.
2285          *
2286          * if private key is not already set, then set it to random value
2287          * and update internal private key.
2288          */
2289         if (!DH_generate_key(dh_key)) {
2290                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2291                 return 0;
2292         }
2293
2294         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
2295                 const BIGNUM *pub_key = NULL;
2296
2297                 OPENSSL_LOG(DEBUG, "%s:%d update public key\n",
2298                                 __func__, __LINE__);
2299
2300                 /* get the generated keys */
2301                 get_dh_pub_key(dh_key, &pub_key);
2302
2303                 /* output public key */
2304                 op->pub_key.length = BN_bn2bin(pub_key,
2305                                 op->pub_key.data);
2306         }
2307
2308         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
2309                 const BIGNUM *priv_key = NULL;
2310
2311                 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n",
2312                                 __func__, __LINE__);
2313
2314                 /* get the generated keys */
2315                 get_dh_priv_key(dh_key, &priv_key);
2316
2317                 /* provide generated private key back to user */
2318                 op->priv_key.length = BN_bn2bin(priv_key,
2319                                 op->priv_key.data);
2320         }
2321
2322         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2323
2324         return 0;
2325 }
2326 #endif
2327
2328 /* process modinv operation */
2329 static int
2330 process_openssl_modinv_op(struct rte_crypto_op *cop,
2331                 struct openssl_asym_session *sess)
2332 {
2333         struct rte_crypto_asym_op *op = cop->asym;
2334         BIGNUM *base = BN_CTX_get(sess->u.m.ctx);
2335         BIGNUM *res = BN_CTX_get(sess->u.m.ctx);
2336
2337         if (unlikely(base == NULL || res == NULL)) {
2338                 BN_free(base);
2339                 BN_free(res);
2340                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2341                 return -1;
2342         }
2343
2344         base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
2345                         op->modinv.base.length, base);
2346
2347         if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) {
2348                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2349                 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data);
2350         } else {
2351                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2352         }
2353
2354         BN_clear(res);
2355         BN_clear(base);
2356
2357         return 0;
2358 }
2359
2360 /* process modexp operation */
2361 static int
2362 process_openssl_modexp_op(struct rte_crypto_op *cop,
2363                 struct openssl_asym_session *sess)
2364 {
2365         struct rte_crypto_asym_op *op = cop->asym;
2366         BIGNUM *base = BN_CTX_get(sess->u.e.ctx);
2367         BIGNUM *res = BN_CTX_get(sess->u.e.ctx);
2368
2369         if (unlikely(base == NULL || res == NULL)) {
2370                 BN_free(base);
2371                 BN_free(res);
2372                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2373                 return -1;
2374         }
2375
2376         base = BN_bin2bn((const unsigned char *)op->modex.base.data,
2377                         op->modex.base.length, base);
2378
2379         if (BN_mod_exp(res, base, sess->u.e.exp,
2380                                 sess->u.e.mod, sess->u.e.ctx)) {
2381                 op->modex.result.length = BN_bn2bin(res, op->modex.result.data);
2382                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2383         } else {
2384                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2385         }
2386
2387         BN_clear(res);
2388         BN_clear(base);
2389
2390         return 0;
2391 }
2392
2393 /* process rsa operations */
2394 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2395 static int
2396 process_openssl_rsa_op_evp(struct rte_crypto_op *cop,
2397                 struct openssl_asym_session *sess)
2398 {
2399         struct rte_crypto_asym_op *op = cop->asym;
2400         uint32_t pad = (op->rsa.padding.type);
2401         uint8_t *tmp;
2402         size_t outlen = 0;
2403         int ret = -1;
2404
2405         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2406         EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx;
2407         if (!rsa_ctx)
2408                 return ret;
2409
2410         switch (pad) {
2411         case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
2412                 pad = RSA_PKCS1_PADDING;
2413                 break;
2414         case RTE_CRYPTO_RSA_PADDING_NONE:
2415                 pad = RSA_NO_PADDING;
2416                 break;
2417         default:
2418                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2419                 OPENSSL_LOG(ERR,
2420                                 "rsa pad type not supported %d\n", pad);
2421                 return ret;
2422         }
2423
2424         switch (op->rsa.op_type) {
2425         case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2426                 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1)
2427                         goto err_rsa;
2428
2429                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2430                         goto err_rsa;
2431
2432                 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen,
2433                                 op->rsa.message.data,
2434                                 op->rsa.message.length) <= 0)
2435                         goto err_rsa;
2436
2437                 if (outlen <= 0)
2438                         goto err_rsa;
2439
2440                 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen,
2441                                 op->rsa.message.data,
2442                                 op->rsa.message.length) <= 0)
2443                         goto err_rsa;
2444                 op->rsa.cipher.length = outlen;
2445
2446                 OPENSSL_LOG(DEBUG,
2447                                 "length of encrypted text %zu\n", outlen);
2448                 break;
2449
2450         case RTE_CRYPTO_ASYM_OP_DECRYPT:
2451                 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1)
2452                         goto err_rsa;
2453
2454                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2455                         goto err_rsa;
2456
2457                 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen,
2458                                 op->rsa.cipher.data,
2459                                 op->rsa.cipher.length) <= 0)
2460                         goto err_rsa;
2461
2462                 if (outlen <= 0)
2463                         goto err_rsa;
2464
2465                 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen,
2466                                 op->rsa.cipher.data,
2467                                 op->rsa.cipher.length) <= 0)
2468                         goto err_rsa;
2469                 op->rsa.message.length = outlen;
2470
2471                 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen);
2472                 break;
2473
2474         case RTE_CRYPTO_ASYM_OP_SIGN:
2475                 if (EVP_PKEY_sign_init(rsa_ctx) <= 0)
2476                         goto err_rsa;
2477
2478                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2479                         goto err_rsa;
2480
2481                 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen,
2482                                 op->rsa.message.data,
2483                                 op->rsa.message.length) <= 0)
2484                         goto err_rsa;
2485                 op->rsa.sign.length = outlen;
2486                 break;
2487
2488         case RTE_CRYPTO_ASYM_OP_VERIFY:
2489                 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
2490                 if (tmp == NULL) {
2491                         OPENSSL_LOG(ERR, "Memory allocation failed");
2492                         goto err_rsa;
2493                 }
2494
2495                 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) {
2496                         rte_free(tmp);
2497                         goto err_rsa;
2498                 }
2499
2500                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) {
2501                         rte_free(tmp);
2502                         goto err_rsa;
2503                 }
2504
2505                 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen,
2506                                 op->rsa.sign.data,
2507                                 op->rsa.sign.length) <= 0) {
2508                         rte_free(tmp);
2509                         goto err_rsa;
2510                 }
2511
2512                 OPENSSL_LOG(DEBUG,
2513                                 "Length of public_decrypt %zu "
2514                                 "length of message %zd\n",
2515                                 outlen, op->rsa.message.length);
2516                 if (CRYPTO_memcmp(tmp, op->rsa.message.data,
2517                                 op->rsa.message.length)) {
2518                         OPENSSL_LOG(ERR, "RSA sign Verification failed");
2519                 }
2520                 rte_free(tmp);
2521                 break;
2522
2523         default:
2524                 /* allow ops with invalid args to be pushed to
2525                  * completion queue
2526                  */
2527                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2528                 goto err_rsa;
2529         }
2530
2531         ret = 0;
2532         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2533 err_rsa:
2534         return ret;
2535
2536 }
2537 #else
2538 static int
2539 process_openssl_rsa_op(struct rte_crypto_op *cop,
2540                 struct openssl_asym_session *sess)
2541 {
2542         int ret = 0;
2543         struct rte_crypto_asym_op *op = cop->asym;
2544         RSA *rsa = sess->u.r.rsa;
2545         uint32_t pad = (op->rsa.padding.type);
2546         uint8_t *tmp;
2547
2548         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2549
2550         switch (pad) {
2551         case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
2552                 pad = RSA_PKCS1_PADDING;
2553                 break;
2554         case RTE_CRYPTO_RSA_PADDING_NONE:
2555                 pad = RSA_NO_PADDING;
2556                 break;
2557         default:
2558                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2559                 OPENSSL_LOG(ERR,
2560                                 "rsa pad type not supported %d\n", pad);
2561                 return 0;
2562         }
2563
2564         switch (op->rsa.op_type) {
2565         case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2566                 ret = RSA_public_encrypt(op->rsa.message.length,
2567                                 op->rsa.message.data,
2568                                 op->rsa.cipher.data,
2569                                 rsa,
2570                                 pad);
2571
2572                 if (ret > 0)
2573                         op->rsa.cipher.length = ret;
2574                 OPENSSL_LOG(DEBUG,
2575                                 "length of encrypted text %d\n", ret);
2576                 break;
2577
2578         case RTE_CRYPTO_ASYM_OP_DECRYPT:
2579                 ret = RSA_private_decrypt(op->rsa.cipher.length,
2580                                 op->rsa.cipher.data,
2581                                 op->rsa.message.data,
2582                                 rsa,
2583                                 pad);
2584                 if (ret > 0)
2585                         op->rsa.message.length = ret;
2586                 break;
2587
2588         case RTE_CRYPTO_ASYM_OP_SIGN:
2589                 ret = RSA_private_encrypt(op->rsa.message.length,
2590                                 op->rsa.message.data,
2591                                 op->rsa.sign.data,
2592                                 rsa,
2593                                 pad);
2594                 if (ret > 0)
2595                         op->rsa.sign.length = ret;
2596                 break;
2597
2598         case RTE_CRYPTO_ASYM_OP_VERIFY:
2599                 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
2600                 if (tmp == NULL) {
2601                         OPENSSL_LOG(ERR, "Memory allocation failed");
2602                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2603                         break;
2604                 }
2605                 ret = RSA_public_decrypt(op->rsa.sign.length,
2606                                 op->rsa.sign.data,
2607                                 tmp,
2608                                 rsa,
2609                                 pad);
2610
2611                 OPENSSL_LOG(DEBUG,
2612                                 "Length of public_decrypt %d "
2613                                 "length of message %zd\n",
2614                                 ret, op->rsa.message.length);
2615                 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data,
2616                                 op->rsa.message.length))) {
2617                         OPENSSL_LOG(ERR, "RSA sign Verification failed");
2618                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2619                 }
2620                 rte_free(tmp);
2621                 break;
2622
2623         default:
2624                 /* allow ops with invalid args to be pushed to
2625                  * completion queue
2626                  */
2627                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2628                 break;
2629         }
2630
2631         if (ret < 0)
2632                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2633
2634         return 0;
2635 }
2636 #endif
2637
2638 static int
2639 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2640                 struct openssl_asym_session *sess)
2641 {
2642         int retval = 0;
2643
2644         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2645
2646         switch (sess->xfrm_type) {
2647         case RTE_CRYPTO_ASYM_XFORM_RSA:
2648 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2649                 retval = process_openssl_rsa_op_evp(op, sess);
2650 # else
2651                 retval = process_openssl_rsa_op(op, sess);
2652 #endif
2653                 break;
2654         case RTE_CRYPTO_ASYM_XFORM_MODEX:
2655                 retval = process_openssl_modexp_op(op, sess);
2656                 break;
2657         case RTE_CRYPTO_ASYM_XFORM_MODINV:
2658                 retval = process_openssl_modinv_op(op, sess);
2659                 break;
2660         case RTE_CRYPTO_ASYM_XFORM_DH:
2661 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2662                 retval = process_openssl_dh_op_evp(op, sess);
2663 # else
2664                 retval = process_openssl_dh_op(op, sess);
2665 #endif
2666                 break;
2667         case RTE_CRYPTO_ASYM_XFORM_DSA:
2668 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2669                 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
2670                         retval = process_openssl_dsa_sign_op_evp(op, sess);
2671                 else if (op->asym->dsa.op_type ==
2672                                 RTE_CRYPTO_ASYM_OP_VERIFY)
2673                         retval =
2674                                 process_openssl_dsa_verify_op_evp(op, sess);
2675 #else
2676                 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
2677                         retval = process_openssl_dsa_sign_op(op, sess);
2678                 else if (op->asym->dsa.op_type ==
2679                                 RTE_CRYPTO_ASYM_OP_VERIFY)
2680                         retval =
2681                                 process_openssl_dsa_verify_op(op, sess);
2682                 else
2683                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2684 #endif
2685                 break;
2686         default:
2687                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2688                 break;
2689         }
2690         if (!retval) {
2691                 /* op processed so push to completion queue as processed */
2692                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2693                 if (retval)
2694                         /* return error if failed to put in completion queue */
2695                         retval = -1;
2696         }
2697
2698         return retval;
2699 }
2700
2701 static void
2702 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst,
2703                 struct rte_crypto_op *op)
2704 {
2705         uint8_t *p_src, *p_dst;
2706
2707         p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
2708         p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
2709
2710         /**
2711          * Copy the content between cipher offset and auth offset
2712          * for generating correct digest.
2713          */
2714         if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
2715                 memcpy(p_dst + op->sym->auth.data.offset,
2716                                 p_src + op->sym->auth.data.offset,
2717                                 op->sym->cipher.data.offset -
2718                                 op->sym->auth.data.offset);
2719 }
2720
2721 /** Process crypto operation for mbuf */
2722 static int
2723 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2724                 struct openssl_session *sess)
2725 {
2726         struct rte_mbuf *msrc, *mdst;
2727         int retval;
2728
2729         msrc = op->sym->m_src;
2730         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
2731
2732         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2733
2734         switch (sess->chain_order) {
2735         case OPENSSL_CHAIN_ONLY_CIPHER:
2736                 process_openssl_cipher_op(op, sess, msrc, mdst);
2737                 break;
2738         case OPENSSL_CHAIN_ONLY_AUTH:
2739                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2740                 break;
2741         case OPENSSL_CHAIN_CIPHER_AUTH:
2742                 process_openssl_cipher_op(op, sess, msrc, mdst);
2743                 /* OOP */
2744                 if (msrc != mdst)
2745                         copy_plaintext(msrc, mdst, op);
2746                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
2747                 break;
2748         case OPENSSL_CHAIN_AUTH_CIPHER:
2749                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2750                 process_openssl_cipher_op(op, sess, msrc, mdst);
2751                 break;
2752         case OPENSSL_CHAIN_COMBINED:
2753                 process_openssl_combined_op(op, sess, msrc, mdst);
2754                 break;
2755         case OPENSSL_CHAIN_CIPHER_BPI:
2756                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
2757                 break;
2758         default:
2759                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
2760                 break;
2761         }
2762
2763         /* Free session if a session-less crypto op */
2764         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
2765                 openssl_reset_session(sess);
2766                 memset(sess, 0, sizeof(struct openssl_session));
2767                 memset(op->sym->session, 0,
2768                         rte_cryptodev_sym_get_existing_header_session_size(
2769                                 op->sym->session));
2770                 rte_mempool_put(qp->sess_mp_priv, sess);
2771                 rte_mempool_put(qp->sess_mp, op->sym->session);
2772                 op->sym->session = NULL;
2773         }
2774
2775         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
2776                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2777
2778         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
2779                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2780         else
2781                 retval = -1;
2782
2783         return retval;
2784 }
2785
2786 /*
2787  *------------------------------------------------------------------------------
2788  * PMD Framework
2789  *------------------------------------------------------------------------------
2790  */
2791
2792 /** Enqueue burst */
2793 static uint16_t
2794 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
2795                 uint16_t nb_ops)
2796 {
2797         void *sess;
2798         struct openssl_qp *qp = queue_pair;
2799         int i, retval;
2800
2801         for (i = 0; i < nb_ops; i++) {
2802                 sess = get_session(qp, ops[i]);
2803                 if (unlikely(sess == NULL))
2804                         goto enqueue_err;
2805
2806                 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
2807                         retval = process_op(qp, ops[i],
2808                                         (struct openssl_session *) sess);
2809                 else
2810                         retval = process_asym_op(qp, ops[i],
2811                                         (struct openssl_asym_session *) sess);
2812                 if (unlikely(retval < 0))
2813                         goto enqueue_err;
2814         }
2815
2816         qp->stats.enqueued_count += i;
2817         return i;
2818
2819 enqueue_err:
2820         qp->stats.enqueue_err_count++;
2821         return i;
2822 }
2823
2824 /** Dequeue burst */
2825 static uint16_t
2826 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
2827                 uint16_t nb_ops)
2828 {
2829         struct openssl_qp *qp = queue_pair;
2830
2831         unsigned int nb_dequeued = 0;
2832
2833         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
2834                         (void **)ops, nb_ops, NULL);
2835         qp->stats.dequeued_count += nb_dequeued;
2836
2837         return nb_dequeued;
2838 }
2839
2840 /** Create OPENSSL crypto device */
2841 static int
2842 cryptodev_openssl_create(const char *name,
2843                         struct rte_vdev_device *vdev,
2844                         struct rte_cryptodev_pmd_init_params *init_params)
2845 {
2846         struct rte_cryptodev *dev;
2847         struct openssl_private *internals;
2848
2849         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2850         if (dev == NULL) {
2851                 OPENSSL_LOG(ERR, "failed to create cryptodev vdev");
2852                 goto init_error;
2853         }
2854
2855         dev->driver_id = cryptodev_driver_id;
2856         dev->dev_ops = rte_openssl_pmd_ops;
2857
2858         /* register rx/tx burst functions for data path */
2859         dev->dequeue_burst = openssl_pmd_dequeue_burst;
2860         dev->enqueue_burst = openssl_pmd_enqueue_burst;
2861
2862         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2863                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2864                         RTE_CRYPTODEV_FF_CPU_AESNI |
2865                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2866                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2867                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2868                         RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
2869                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP |
2870                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT |
2871                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
2872
2873         internals = dev->data->dev_private;
2874
2875         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
2876
2877         rte_cryptodev_pmd_probing_finish(dev);
2878
2879 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2880         /* Load legacy provider
2881          * Some algorithms are no longer available in earlier version of openssl,
2882          * unless the legacy provider explicitly loaded. e.g. DES
2883          */
2884         ossl_legacy_provider_load();
2885 # endif
2886         return 0;
2887
2888 init_error:
2889         OPENSSL_LOG(ERR, "driver %s: create failed",
2890                         init_params->name);
2891
2892         cryptodev_openssl_remove(vdev);
2893         return -EFAULT;
2894 }
2895
2896 /** Initialise OPENSSL crypto device */
2897 static int
2898 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
2899 {
2900         struct rte_cryptodev_pmd_init_params init_params = {
2901                 "",
2902                 sizeof(struct openssl_private),
2903                 rte_socket_id(),
2904                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2905         };
2906         const char *name;
2907         const char *input_args;
2908
2909         name = rte_vdev_device_name(vdev);
2910         if (name == NULL)
2911                 return -EINVAL;
2912         input_args = rte_vdev_device_args(vdev);
2913
2914         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2915
2916         return cryptodev_openssl_create(name, vdev, &init_params);
2917 }
2918
2919 /** Uninitialise OPENSSL crypto device */
2920 static int
2921 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
2922 {
2923         struct rte_cryptodev *cryptodev;
2924         const char *name;
2925
2926         name = rte_vdev_device_name(vdev);
2927         if (name == NULL)
2928                 return -EINVAL;
2929
2930         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2931         if (cryptodev == NULL)
2932                 return -ENODEV;
2933
2934 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2935         ossl_legacy_provider_unload();
2936 # endif
2937         return rte_cryptodev_pmd_destroy(cryptodev);
2938 }
2939
2940 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
2941         .probe = cryptodev_openssl_probe,
2942         .remove = cryptodev_openssl_remove
2943 };
2944
2945 static struct cryptodev_driver openssl_crypto_drv;
2946
2947 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
2948         cryptodev_openssl_pmd_drv);
2949 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
2950         "max_nb_queue_pairs=<int> "
2951         "socket_id=<int>");
2952 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
2953                 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);
2954 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO);