drivers/crypto/openssl/rte_openssl_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #define OPENSSL_API_COMPAT 0x10100000L
   6
   7 #include <rte_common.h>
   8 #include <rte_hexdump.h>
   9 #include <rte_cryptodev.h>
  10 #include <cryptodev_pmd.h>
  11 #include <rte_bus_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cpuflags.h>
  14
  15 #include <openssl/hmac.h>
  16 #include <openssl/evp.h>
  17
  18 #include "openssl_pmd_private.h"
  19 #include "compat.h"
  20
  21 #define DES_BLOCK_SIZE 8
  22
  23 static uint8_t cryptodev_driver_id;
  24
  25 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  26 static HMAC_CTX *HMAC_CTX_new(void)
  27 {
  28         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  29
  30         if (ctx != NULL)
  31                 HMAC_CTX_init(ctx);
  32         return ctx;
  33 }
  34
  35 static void HMAC_CTX_free(HMAC_CTX *ctx)
  36 {
  37         if (ctx != NULL) {
  38                 HMAC_CTX_cleanup(ctx);
  39                 OPENSSL_free(ctx);
  40         }
  41 }
  42 #endif
  43
  44 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
  45
  46 #include <openssl/provider.h>
  47 #include <openssl/core_names.h>
  48 #include <openssl/param_build.h>
  49
  50 #define MAX_OSSL_ALGO_NAME_SIZE         16
  51
  52 OSSL_PROVIDER *legacy;
  53 OSSL_PROVIDER *deflt;
  54
  55 static void ossl_legacy_provider_load(void)
  56 {
  57         /* Load Multiple providers into the default (NULL) library context */
  58         legacy = OSSL_PROVIDER_load(NULL, "legacy");
  59         if (legacy == NULL) {
  60                 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n");
  61                 return;
  62         }
  63
  64         deflt = OSSL_PROVIDER_load(NULL, "default");
  65         if (deflt == NULL) {
  66                 OPENSSL_LOG(ERR, "Failed to load Default provider\n");
  67                 OSSL_PROVIDER_unload(legacy);
  68                 return;
  69         }
  70 }
  71
  72 static void ossl_legacy_provider_unload(void)
  73 {
  74         OSSL_PROVIDER_unload(legacy);
  75         OSSL_PROVIDER_unload(deflt);
  76 }
  77
  78 static __rte_always_inline const char *
  79 digest_name_get(enum rte_crypto_auth_algorithm algo)
  80 {
  81         switch (algo) {
  82         case RTE_CRYPTO_AUTH_MD5_HMAC:
  83                 return OSSL_DIGEST_NAME_MD5;
  84         case RTE_CRYPTO_AUTH_SHA1_HMAC:
  85                 return OSSL_DIGEST_NAME_SHA1;
  86         case RTE_CRYPTO_AUTH_SHA224_HMAC:
  87                 return OSSL_DIGEST_NAME_SHA2_224;
  88         case RTE_CRYPTO_AUTH_SHA256_HMAC:
  89                 return OSSL_DIGEST_NAME_SHA2_256;
  90         case RTE_CRYPTO_AUTH_SHA384_HMAC:
  91                 return OSSL_DIGEST_NAME_SHA2_384;
  92         case RTE_CRYPTO_AUTH_SHA512_HMAC:
  93                 return OSSL_DIGEST_NAME_SHA2_512;
  94         default:
  95                 return NULL;
  96         }
  97 }
  98 #endif
  99
 100 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
 101
 102 /*----------------------------------------------------------------------------*/
 103
 104 /**
 105  * Increment counter by 1
 106  * Counter is 64 bit array, big-endian
 107  */
 108 static void
 109 ctr_inc(uint8_t *ctr)
 110 {
 111         uint64_t *ctr64 = (uint64_t *)ctr;
 112
 113         *ctr64 = __builtin_bswap64(*ctr64);
 114         (*ctr64)++;
 115         *ctr64 = __builtin_bswap64(*ctr64);
 116 }
 117
 118 /*
 119  *------------------------------------------------------------------------------
 120  * Session Prepare
 121  *------------------------------------------------------------------------------
 122  */
 123
 124 /** Get xform chain order */
 125 static enum openssl_chain_order
 126 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
 127 {
 128         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
 129
 130         if (xform != NULL) {
 131                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 132                         if (xform->next == NULL)
 133                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
 134                         else if (xform->next->type ==
 135                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
 136                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
 137                 }
 138                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 139                         if (xform->next == NULL)
 140                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
 141                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 142                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
 143                 }
 144                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 145                         res = OPENSSL_CHAIN_COMBINED;
 146         }
 147
 148         return res;
 149 }
 150
 151 /** Get session cipher key from input cipher key */
 152 static void
 153 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key)
 154 {
 155         memcpy(session_key, input_key, keylen);
 156 }
 157
 158 /** Get key ede 24 bytes standard from input key */
 159 static int
 160 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede)
 161 {
 162         int res = 0;
 163
 164         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 165         switch (keylen) {
 166         case 24:
 167                 memcpy(key_ede, key, 24);
 168                 break;
 169         case 16:
 170                 /* K3 = K1 */
 171                 memcpy(key_ede, key, 16);
 172                 memcpy(key_ede + 16, key, 8);
 173                 break;
 174         case 8:
 175                 /* K1 = K2 = K3 (DES compatibility) */
 176                 memcpy(key_ede, key, 8);
 177                 memcpy(key_ede + 8, key, 8);
 178                 memcpy(key_ede + 16, key, 8);
 179                 break;
 180         default:
 181                 OPENSSL_LOG(ERR, "Unsupported key size");
 182                 res = -EINVAL;
 183         }
 184
 185         return res;
 186 }
 187
 188 /** Get adequate openssl function for input cipher algorithm */
 189 static uint8_t
 190 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 191                 const EVP_CIPHER **algo)
 192 {
 193         int res = 0;
 194
 195         if (algo != NULL) {
 196                 switch (sess_algo) {
 197                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 198                         switch (keylen) {
 199                         case 8:
 200                                 *algo = EVP_des_cbc();
 201                                 break;
 202                         case 16:
 203                                 *algo = EVP_des_ede_cbc();
 204                                 break;
 205                         case 24:
 206                                 *algo = EVP_des_ede3_cbc();
 207                                 break;
 208                         default:
 209                                 res = -EINVAL;
 210                         }
 211                         break;
 212                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 213                         break;
 214                 case RTE_CRYPTO_CIPHER_AES_CBC:
 215                         switch (keylen) {
 216                         case 16:
 217                                 *algo = EVP_aes_128_cbc();
 218                                 break;
 219                         case 24:
 220                                 *algo = EVP_aes_192_cbc();
 221                                 break;
 222                         case 32:
 223                                 *algo = EVP_aes_256_cbc();
 224                                 break;
 225                         default:
 226                                 res = -EINVAL;
 227                         }
 228                         break;
 229                 case RTE_CRYPTO_CIPHER_AES_CTR:
 230                         switch (keylen) {
 231                         case 16:
 232                                 *algo = EVP_aes_128_ctr();
 233                                 break;
 234                         case 24:
 235                                 *algo = EVP_aes_192_ctr();
 236                                 break;
 237                         case 32:
 238                                 *algo = EVP_aes_256_ctr();
 239                                 break;
 240                         default:
 241                                 res = -EINVAL;
 242                         }
 243                         break;
 244                 default:
 245                         res = -EINVAL;
 246                         break;
 247                 }
 248         } else {
 249                 res = -EINVAL;
 250         }
 251
 252         return res;
 253 }
 254
 255 /** Get adequate openssl function for input auth algorithm */
 256 static uint8_t
 257 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 258                 const EVP_MD **algo)
 259 {
 260         int res = 0;
 261
 262         if (algo != NULL) {
 263                 switch (sessalgo) {
 264                 case RTE_CRYPTO_AUTH_MD5:
 265                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 266                         *algo = EVP_md5();
 267                         break;
 268                 case RTE_CRYPTO_AUTH_SHA1:
 269                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 270                         *algo = EVP_sha1();
 271                         break;
 272                 case RTE_CRYPTO_AUTH_SHA224:
 273                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 274                         *algo = EVP_sha224();
 275                         break;
 276                 case RTE_CRYPTO_AUTH_SHA256:
 277                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 278                         *algo = EVP_sha256();
 279                         break;
 280                 case RTE_CRYPTO_AUTH_SHA384:
 281                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 282                         *algo = EVP_sha384();
 283                         break;
 284                 case RTE_CRYPTO_AUTH_SHA512:
 285                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 286                         *algo = EVP_sha512();
 287                         break;
 288                 default:
 289                         res = -EINVAL;
 290                         break;
 291                 }
 292         } else {
 293                 res = -EINVAL;
 294         }
 295
 296         return res;
 297 }
 298
 299 /** Get adequate openssl function for input cipher algorithm */
 300 static uint8_t
 301 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 302                 const EVP_CIPHER **algo)
 303 {
 304         int res = 0;
 305
 306         if (algo != NULL) {
 307                 switch (sess_algo) {
 308                 case RTE_CRYPTO_AEAD_AES_GCM:
 309                         switch (keylen) {
 310                         case 16:
 311                                 *algo = EVP_aes_128_gcm();
 312                                 break;
 313                         case 24:
 314                                 *algo = EVP_aes_192_gcm();
 315                                 break;
 316                         case 32:
 317                                 *algo = EVP_aes_256_gcm();
 318                                 break;
 319                         default:
 320                                 res = -EINVAL;
 321                         }
 322                         break;
 323                 case RTE_CRYPTO_AEAD_AES_CCM:
 324                         switch (keylen) {
 325                         case 16:
 326                                 *algo = EVP_aes_128_ccm();
 327                                 break;
 328                         case 24:
 329                                 *algo = EVP_aes_192_ccm();
 330                                 break;
 331                         case 32:
 332                                 *algo = EVP_aes_256_ccm();
 333                                 break;
 334                         default:
 335                                 res = -EINVAL;
 336                         }
 337                         break;
 338                 default:
 339                         res = -EINVAL;
 340                         break;
 341                 }
 342         } else {
 343                 res = -EINVAL;
 344         }
 345
 346         return res;
 347 }
 348
 349 /* Set session AEAD encryption parameters */
 350 static int
 351 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
 352                 enum rte_crypto_aead_algorithm algo,
 353                 uint8_t tag_len, const uint8_t *key)
 354 {
 355         int iv_type = 0;
 356         unsigned int do_ccm;
 357
 358         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 359         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 360
 361         /* Select AEAD algo */
 362         switch (algo) {
 363         case RTE_CRYPTO_AEAD_AES_GCM:
 364                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 365                 if (tag_len != 16)
 366                         return -EINVAL;
 367                 do_ccm = 0;
 368                 break;
 369         case RTE_CRYPTO_AEAD_AES_CCM:
 370                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 371                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 372                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 373                         return -EINVAL;
 374                 do_ccm = 1;
 375                 break;
 376         default:
 377                 return -ENOTSUP;
 378         }
 379
 380         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 381         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 382
 383         if (get_aead_algo(algo, sess->cipher.key.length,
 384                         &sess->cipher.evp_algo) != 0)
 385                 return -EINVAL;
 386
 387         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 388
 389         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 390
 391         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 392                         NULL, NULL, NULL) <= 0)
 393                 return -EINVAL;
 394
 395         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
 396                         NULL) <= 0)
 397                 return -EINVAL;
 398
 399         if (do_ccm)
 400                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 401                                 tag_len, NULL);
 402
 403         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 404                 return -EINVAL;
 405
 406         return 0;
 407 }
 408
 409 /* Set session AEAD decryption parameters */
 410 static int
 411 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
 412                 enum rte_crypto_aead_algorithm algo,
 413                 uint8_t tag_len, const uint8_t *key)
 414 {
 415         int iv_type = 0;
 416         unsigned int do_ccm = 0;
 417
 418         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 419         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 420
 421         /* Select AEAD algo */
 422         switch (algo) {
 423         case RTE_CRYPTO_AEAD_AES_GCM:
 424                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 425                 if (tag_len != 16)
 426                         return -EINVAL;
 427                 break;
 428         case RTE_CRYPTO_AEAD_AES_CCM:
 429                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 430                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 431                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 432                         return -EINVAL;
 433                 do_ccm = 1;
 434                 break;
 435         default:
 436                 return -ENOTSUP;
 437         }
 438
 439         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 440         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 441
 442         if (get_aead_algo(algo, sess->cipher.key.length,
 443                         &sess->cipher.evp_algo) != 0)
 444                 return -EINVAL;
 445
 446         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 447
 448         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 449
 450         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 451                         NULL, NULL, NULL) <= 0)
 452                 return -EINVAL;
 453
 454         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
 455                         sess->iv.length, NULL) <= 0)
 456                 return -EINVAL;
 457
 458         if (do_ccm)
 459                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 460                                 tag_len, NULL);
 461
 462         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 463                 return -EINVAL;
 464
 465         return 0;
 466 }
 467
 468 /** Set session cipher parameters */
 469 static int
 470 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 471                 const struct rte_crypto_sym_xform *xform)
 472 {
 473         /* Select cipher direction */
 474         sess->cipher.direction = xform->cipher.op;
 475         /* Select cipher key */
 476         sess->cipher.key.length = xform->cipher.key.length;
 477
 478         /* Set IV parameters */
 479         sess->iv.offset = xform->cipher.iv.offset;
 480         sess->iv.length = xform->cipher.iv.length;
 481
 482         /* Select cipher algo */
 483         switch (xform->cipher.algo) {
 484         case RTE_CRYPTO_CIPHER_3DES_CBC:
 485         case RTE_CRYPTO_CIPHER_AES_CBC:
 486         case RTE_CRYPTO_CIPHER_AES_CTR:
 487                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 488                 sess->cipher.algo = xform->cipher.algo;
 489                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 490
 491                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 492                                 &sess->cipher.evp_algo) != 0)
 493                         return -EINVAL;
 494
 495                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 496                         sess->cipher.key.data);
 497                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 498                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 499                                         sess->cipher.evp_algo,
 500                                         NULL, xform->cipher.key.data,
 501                                         NULL) != 1) {
 502                                 return -EINVAL;
 503                         }
 504                 } else if (sess->cipher.direction ==
 505                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 506                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 507                                         sess->cipher.evp_algo,
 508                                         NULL, xform->cipher.key.data,
 509                                         NULL) != 1) {
 510                                 return -EINVAL;
 511                         }
 512                 }
 513
 514                 break;
 515
 516         case RTE_CRYPTO_CIPHER_3DES_CTR:
 517                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 518                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 519
 520                 if (get_cipher_key_ede(xform->cipher.key.data,
 521                                 sess->cipher.key.length,
 522                                 sess->cipher.key.data) != 0)
 523                         return -EINVAL;
 524                 break;
 525
 526         case RTE_CRYPTO_CIPHER_DES_CBC:
 527                 sess->cipher.algo = xform->cipher.algo;
 528                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 529                 sess->cipher.evp_algo = EVP_des_cbc();
 530
 531                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 532                         sess->cipher.key.data);
 533                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 534                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 535                                         sess->cipher.evp_algo,
 536                                         NULL, xform->cipher.key.data,
 537                                         NULL) != 1) {
 538                                 return -EINVAL;
 539                         }
 540                 } else if (sess->cipher.direction ==
 541                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 542                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 543                                         sess->cipher.evp_algo,
 544                                         NULL, xform->cipher.key.data,
 545                                         NULL) != 1) {
 546                                 return -EINVAL;
 547                         }
 548                 }
 549
 550                 break;
 551
 552         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 553                 sess->cipher.algo = xform->cipher.algo;
 554                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 555                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 556                 sess->cipher.evp_algo = EVP_des_cbc();
 557
 558                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 559                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 560                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 561                                 NULL, xform->cipher.key.data, 0) != 1)
 562                         return -EINVAL;
 563
 564                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 565                         sess->cipher.key.data);
 566                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 567                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 568                                         sess->cipher.evp_algo,
 569                                         NULL, xform->cipher.key.data,
 570                                         NULL) != 1) {
 571                                 return -EINVAL;
 572                         }
 573                 } else if (sess->cipher.direction ==
 574                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 575                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 576                                         sess->cipher.evp_algo,
 577                                         NULL, xform->cipher.key.data,
 578                                         NULL) != 1) {
 579                                 return -EINVAL;
 580                         }
 581                 }
 582
 583                 break;
 584         default:
 585                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 586                 return -ENOTSUP;
 587         }
 588
 589         return 0;
 590 }
 591
 592 /* Set session auth parameters */
 593 static int
 594 openssl_set_session_auth_parameters(struct openssl_session *sess,
 595                 const struct rte_crypto_sym_xform *xform)
 596 {
 597         /* Select auth generate/verify */
 598         sess->auth.operation = xform->auth.op;
 599         sess->auth.algo = xform->auth.algo;
 600
 601         sess->auth.digest_length = xform->auth.digest_length;
 602
 603         /* Select auth algo */
 604         switch (xform->auth.algo) {
 605         case RTE_CRYPTO_AUTH_AES_GMAC:
 606                 /*
 607                  * OpenSSL requires GMAC to be a GCM operation
 608                  * with no cipher data length
 609                  */
 610                 sess->cipher.key.length = xform->auth.key.length;
 611
 612                 /* Set IV parameters */
 613                 sess->iv.offset = xform->auth.iv.offset;
 614                 sess->iv.length = xform->auth.iv.length;
 615
 616                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 617                         return openssl_set_sess_aead_enc_param(sess,
 618                                                 RTE_CRYPTO_AEAD_AES_GCM,
 619                                                 xform->auth.digest_length,
 620                                                 xform->auth.key.data);
 621                 else
 622                         return openssl_set_sess_aead_dec_param(sess,
 623                                                 RTE_CRYPTO_AEAD_AES_GCM,
 624                                                 xform->auth.digest_length,
 625                                                 xform->auth.key.data);
 626                 break;
 627
 628         case RTE_CRYPTO_AUTH_MD5:
 629         case RTE_CRYPTO_AUTH_SHA1:
 630         case RTE_CRYPTO_AUTH_SHA224:
 631         case RTE_CRYPTO_AUTH_SHA256:
 632         case RTE_CRYPTO_AUTH_SHA384:
 633         case RTE_CRYPTO_AUTH_SHA512:
 634                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 635                 if (get_auth_algo(xform->auth.algo,
 636                                 &sess->auth.auth.evp_algo) != 0)
 637                         return -EINVAL;
 638                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 639                 break;
 640
 641 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
 642         case RTE_CRYPTO_AUTH_MD5_HMAC:
 643         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 644         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 645         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 646         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 647         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 648                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 649
 650                 OSSL_PARAM params[2];
 651                 const char *algo;
 652                 algo = digest_name_get(xform->auth.algo);
 653                 if (!algo)
 654                         return -EINVAL;
 655                 char algo_name[MAX_OSSL_ALGO_NAME_SIZE];
 656                 rte_memcpy(algo_name, algo, (sizeof(algo)+1));
 657
 658                 EVP_MAC *mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
 659                 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac);
 660                 EVP_MAC_free(mac);
 661                 if (get_auth_algo(xform->auth.algo,
 662                                 &sess->auth.hmac.evp_algo) != 0)
 663                         return -EINVAL;
 664
 665                 params[0] = OSSL_PARAM_construct_utf8_string("digest",
 666                                         algo_name, 0);
 667                 params[1] = OSSL_PARAM_construct_end();
 668                 if (EVP_MAC_init(sess->auth.hmac.ctx,
 669                                 xform->auth.key.data,
 670                                 xform->auth.key.length,
 671                                 params) != 1)
 672                         return -EINVAL;
 673                 break;
 674 # else
 675         case RTE_CRYPTO_AUTH_MD5_HMAC:
 676         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 677         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 678         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 679         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 680         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 681                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 682                 sess->auth.hmac.ctx = HMAC_CTX_new();
 683                 if (get_auth_algo(xform->auth.algo,
 684                                 &sess->auth.hmac.evp_algo) != 0)
 685                         return -EINVAL;
 686
 687                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 688                                 xform->auth.key.data,
 689                                 xform->auth.key.length,
 690                                 sess->auth.hmac.evp_algo, NULL) != 1)
 691                         return -EINVAL;
 692                 break;
 693 # endif
 694         default:
 695                 return -ENOTSUP;
 696         }
 697
 698         return 0;
 699 }
 700
 701 /* Set session AEAD parameters */
 702 static int
 703 openssl_set_session_aead_parameters(struct openssl_session *sess,
 704                 const struct rte_crypto_sym_xform *xform)
 705 {
 706         /* Select cipher key */
 707         sess->cipher.key.length = xform->aead.key.length;
 708
 709         /* Set IV parameters */
 710         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 711                 /*
 712                  * For AES-CCM, the actual IV is placed
 713                  * one byte after the start of the IV field,
 714                  * according to the API.
 715                  */
 716                 sess->iv.offset = xform->aead.iv.offset + 1;
 717         else
 718                 sess->iv.offset = xform->aead.iv.offset;
 719
 720         sess->iv.length = xform->aead.iv.length;
 721
 722         sess->auth.aad_length = xform->aead.aad_length;
 723         sess->auth.digest_length = xform->aead.digest_length;
 724
 725         sess->aead_algo = xform->aead.algo;
 726         /* Select cipher direction */
 727         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 728                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
 729                                 xform->aead.digest_length, xform->aead.key.data);
 730         else
 731                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
 732                                 xform->aead.digest_length, xform->aead.key.data);
 733 }
 734
 735 /** Parse crypto xform chain and set private session parameters */
 736 int
 737 openssl_set_session_parameters(struct openssl_session *sess,
 738                 const struct rte_crypto_sym_xform *xform)
 739 {
 740         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 741         const struct rte_crypto_sym_xform *auth_xform = NULL;
 742         const struct rte_crypto_sym_xform *aead_xform = NULL;
 743         int ret;
 744
 745         sess->chain_order = openssl_get_chain_order(xform);
 746         switch (sess->chain_order) {
 747         case OPENSSL_CHAIN_ONLY_CIPHER:
 748                 cipher_xform = xform;
 749                 break;
 750         case OPENSSL_CHAIN_ONLY_AUTH:
 751                 auth_xform = xform;
 752                 break;
 753         case OPENSSL_CHAIN_CIPHER_AUTH:
 754                 cipher_xform = xform;
 755                 auth_xform = xform->next;
 756                 break;
 757         case OPENSSL_CHAIN_AUTH_CIPHER:
 758                 auth_xform = xform;
 759                 cipher_xform = xform->next;
 760                 break;
 761         case OPENSSL_CHAIN_COMBINED:
 762                 aead_xform = xform;
 763                 break;
 764         default:
 765                 return -EINVAL;
 766         }
 767
 768         /* Default IV length = 0 */
 769         sess->iv.length = 0;
 770
 771         /* cipher_xform must be check before auth_xform */
 772         if (cipher_xform) {
 773                 ret = openssl_set_session_cipher_parameters(
 774                                 sess, cipher_xform);
 775                 if (ret != 0) {
 776                         OPENSSL_LOG(ERR,
 777                                 "Invalid/unsupported cipher parameters");
 778                         return ret;
 779                 }
 780         }
 781
 782         if (auth_xform) {
 783                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 784                 if (ret != 0) {
 785                         OPENSSL_LOG(ERR,
 786                                 "Invalid/unsupported auth parameters");
 787                         return ret;
 788                 }
 789         }
 790
 791         if (aead_xform) {
 792                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 793                 if (ret != 0) {
 794                         OPENSSL_LOG(ERR,
 795                                 "Invalid/unsupported AEAD parameters");
 796                         return ret;
 797                 }
 798         }
 799
 800         return 0;
 801 }
 802
 803 /** Reset private session parameters */
 804 void
 805 openssl_reset_session(struct openssl_session *sess)
 806 {
 807         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 808
 809         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 810                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 811
 812         switch (sess->auth.mode) {
 813         case OPENSSL_AUTH_AS_AUTH:
 814                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 815                 break;
 816         case OPENSSL_AUTH_AS_HMAC:
 817                 EVP_PKEY_free(sess->auth.hmac.pkey);
 818 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
 819                 EVP_MAC_CTX_free(sess->auth.hmac.ctx);
 820 # else
 821                 HMAC_CTX_free(sess->auth.hmac.ctx);
 822 # endif
 823                 break;
 824         default:
 825                 break;
 826         }
 827 }
 828
 829 /** Provide session for operation */
 830 static void *
 831 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 832 {
 833         struct openssl_session *sess = NULL;
 834         struct openssl_asym_session *asym_sess = NULL;
 835
 836         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 837                 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 838                         /* get existing session */
 839                         if (likely(op->sym->session != NULL))
 840                                 sess = (struct openssl_session *)
 841                                                 get_sym_session_private_data(
 842                                                 op->sym->session,
 843                                                 cryptodev_driver_id);
 844                 } else {
 845                         if (likely(op->asym->session != NULL))
 846                                 asym_sess = (struct openssl_asym_session *)
 847                                                 op->asym->session->sess_private_data;
 848                         if (asym_sess == NULL)
 849                                 op->status =
 850                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 851                         return asym_sess;
 852                 }
 853         } else {
 854                 /* sessionless asymmetric not supported */
 855                 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
 856                         return NULL;
 857
 858                 /* provide internal session */
 859                 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
 860                 void *_sess_private_data = NULL;
 861
 862                 if (_sess == NULL)
 863                         return NULL;
 864
 865                 if (rte_mempool_get(qp->sess_mp_priv,
 866                                 (void **)&_sess_private_data))
 867                         return NULL;
 868
 869                 sess = (struct openssl_session *)_sess_private_data;
 870
 871                 if (unlikely(openssl_set_session_parameters(sess,
 872                                 op->sym->xform) != 0)) {
 873                         rte_mempool_put(qp->sess_mp, _sess);
 874                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 875                         sess = NULL;
 876                 }
 877                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 878                 set_sym_session_private_data(op->sym->session,
 879                                 cryptodev_driver_id, _sess_private_data);
 880         }
 881
 882         if (sess == NULL)
 883                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 884
 885         return sess;
 886 }
 887
 888 /*
 889  *------------------------------------------------------------------------------
 890  * Process Operations
 891  *------------------------------------------------------------------------------
 892  */
 893 static inline int
 894 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 895                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
 896 {
 897         struct rte_mbuf *m;
 898         int dstlen;
 899         int l, n = srclen;
 900         uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
 901
 902         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 903                         m = m->next)
 904                 offset -= rte_pktmbuf_data_len(m);
 905
 906         if (m == 0)
 907                 return -1;
 908
 909         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 910         if (inplace)
 911                 *dst = src;
 912
 913         l = rte_pktmbuf_data_len(m) - offset;
 914         if (srclen <= l) {
 915                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 916                         return -1;
 917                 *dst += l;
 918                 return 0;
 919         }
 920
 921         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 922                 return -1;
 923
 924         *dst += dstlen;
 925         n -= l;
 926
 927         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 928                 uint8_t diff = l - dstlen, rem;
 929
 930                 src = rte_pktmbuf_mtod(m, uint8_t *);
 931                 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
 932                 if (diff && inplace) {
 933                         rem = RTE_MIN(l,
 934                                 (EVP_CIPHER_CTX_block_size(ctx) - diff));
 935                         if (EVP_EncryptUpdate(ctx, temp,
 936                                                 &dstlen, src, rem) <= 0)
 937                                 return -1;
 938                         n -= rem;
 939                         rte_memcpy(*dst, temp, diff);
 940                         rte_memcpy(src, temp + diff, rem);
 941                         src += rem;
 942                         l -= rem;
 943                 }
 944                 if (inplace)
 945                         *dst = src;
 946                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 947                         return -1;
 948                 *dst += dstlen;
 949                 n -= l;
 950         }
 951
 952         return 0;
 953 }
 954
 955 static inline int
 956 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 957                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
 958 {
 959         struct rte_mbuf *m;
 960         int dstlen;
 961         int l, n = srclen;
 962         uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
 963
 964         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 965                         m = m->next)
 966                 offset -= rte_pktmbuf_data_len(m);
 967
 968         if (m == 0)
 969                 return -1;
 970
 971         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 972         if (inplace)
 973                 *dst = src;
 974
 975         l = rte_pktmbuf_data_len(m) - offset;
 976         if (srclen <= l) {
 977                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 978                         return -1;
 979                 *dst += l;
 980                 return 0;
 981         }
 982
 983         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 984                 return -1;
 985
 986         *dst += dstlen;
 987         n -= l;
 988
 989         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 990                 uint8_t diff = l - dstlen, rem;
 991
 992                 src = rte_pktmbuf_mtod(m, uint8_t *);
 993                 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
 994                 if (diff && inplace) {
 995                         rem = RTE_MIN(l,
 996                                 (EVP_CIPHER_CTX_block_size(ctx) - diff));
 997                         if (EVP_DecryptUpdate(ctx, temp,
 998                                                 &dstlen, src, rem) <= 0)
 999                                 return -1;
1000                         n -= rem;
1001                         rte_memcpy(*dst, temp, diff);
1002                         rte_memcpy(src, temp + diff, rem);
1003                         src += rem;
1004                         l -= rem;
1005                 }
1006                 if (inplace)
1007                         *dst = src;
1008                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
1009                         return -1;
1010                 *dst += dstlen;
1011                 n -= l;
1012         }
1013
1014         return 0;
1015 }
1016
1017 /** Process standard openssl cipher encryption */
1018 static int
1019 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1020                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1021                 uint8_t inplace)
1022 {
1023         int totlen;
1024
1025         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1026                 goto process_cipher_encrypt_err;
1027
1028         EVP_CIPHER_CTX_set_padding(ctx, 0);
1029
1030         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1031                         srclen, ctx, inplace))
1032                 goto process_cipher_encrypt_err;
1033
1034         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
1035                 goto process_cipher_encrypt_err;
1036
1037         return 0;
1038
1039 process_cipher_encrypt_err:
1040         OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed");
1041         return -EINVAL;
1042 }
1043
1044 /** Process standard openssl cipher encryption */
1045 static int
1046 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
1047                 uint8_t *iv, int srclen,
1048                 EVP_CIPHER_CTX *ctx)
1049 {
1050         uint8_t i;
1051         uint8_t encrypted_iv[DES_BLOCK_SIZE];
1052         int encrypted_ivlen;
1053
1054         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
1055                         iv, DES_BLOCK_SIZE) <= 0)
1056                 goto process_cipher_encrypt_err;
1057
1058         for (i = 0; i < srclen; i++)
1059                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
1060
1061         return 0;
1062
1063 process_cipher_encrypt_err:
1064         OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed");
1065         return -EINVAL;
1066 }
1067 /** Process standard openssl cipher decryption */
1068 static int
1069 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1070                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1071                 uint8_t inplace)
1072 {
1073         int totlen;
1074
1075         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1076                 goto process_cipher_decrypt_err;
1077
1078         EVP_CIPHER_CTX_set_padding(ctx, 0);
1079
1080         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1081                         srclen, ctx, inplace))
1082                 goto process_cipher_decrypt_err;
1083
1084         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
1085                 goto process_cipher_decrypt_err;
1086         return 0;
1087
1088 process_cipher_decrypt_err:
1089         OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed");
1090         return -EINVAL;
1091 }
1092
1093 /** Process cipher des 3 ctr encryption, decryption algorithm */
1094 static int
1095 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
1096                 int offset, uint8_t *iv, uint8_t *key, int srclen,
1097                 EVP_CIPHER_CTX *ctx)
1098 {
1099         uint8_t ebuf[8], ctr[8];
1100         int unused, n;
1101         struct rte_mbuf *m;
1102         uint8_t *src;
1103         int l;
1104
1105         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1106                         m = m->next)
1107                 offset -= rte_pktmbuf_data_len(m);
1108
1109         if (m == 0)
1110                 goto process_cipher_des3ctr_err;
1111
1112         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1113         l = rte_pktmbuf_data_len(m) - offset;
1114
1115         /* We use 3DES encryption also for decryption.
1116          * IV is not important for 3DES ecb
1117          */
1118         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
1119                 goto process_cipher_des3ctr_err;
1120
1121         memcpy(ctr, iv, 8);
1122
1123         for (n = 0; n < srclen; n++) {
1124                 if (n % 8 == 0) {
1125                         if (EVP_EncryptUpdate(ctx,
1126                                         (unsigned char *)&ebuf, &unused,
1127                                         (const unsigned char *)&ctr, 8) <= 0)
1128                                 goto process_cipher_des3ctr_err;
1129                         ctr_inc(ctr);
1130                 }
1131                 dst[n] = *(src++) ^ ebuf[n % 8];
1132
1133                 l--;
1134                 if (!l) {
1135                         m = m->next;
1136                         if (m) {
1137                                 src = rte_pktmbuf_mtod(m, uint8_t *);
1138                                 l = rte_pktmbuf_data_len(m);
1139                         }
1140                 }
1141         }
1142
1143         return 0;
1144
1145 process_cipher_des3ctr_err:
1146         OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed");
1147         return -EINVAL;
1148 }
1149
1150 /** Process AES-GCM encrypt algorithm */
1151 static int
1152 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1153                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1154                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1155 {
1156         int len = 0, unused = 0;
1157         uint8_t empty[] = {};
1158
1159         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1160                 goto process_auth_encryption_gcm_err;
1161
1162         if (aadlen > 0)
1163                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1164                         goto process_auth_encryption_gcm_err;
1165
1166         if (srclen > 0)
1167                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1168                                 srclen, ctx, 0))
1169                         goto process_auth_encryption_gcm_err;
1170
1171         /* Workaround open ssl bug in version less then 1.0.1f */
1172         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1173                 goto process_auth_encryption_gcm_err;
1174
1175         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1176                 goto process_auth_encryption_gcm_err;
1177
1178         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1179                 goto process_auth_encryption_gcm_err;
1180
1181         return 0;
1182
1183 process_auth_encryption_gcm_err:
1184         OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed");
1185         return -EINVAL;
1186 }
1187
1188 /** Process AES-CCM encrypt algorithm */
1189 static int
1190 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1191                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1192                 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1193 {
1194         int len = 0;
1195
1196         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1197                 goto process_auth_encryption_ccm_err;
1198
1199         if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1200                 goto process_auth_encryption_ccm_err;
1201
1202         if (aadlen > 0)
1203                 /*
1204                  * For AES-CCM, the actual AAD is placed
1205                  * 18 bytes after the start of the AAD field,
1206                  * according to the API.
1207                  */
1208                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1209                         goto process_auth_encryption_ccm_err;
1210
1211         if (srclen >= 0)
1212                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1213                                 srclen, ctx, 0))
1214                         goto process_auth_encryption_ccm_err;
1215
1216         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1217                 goto process_auth_encryption_ccm_err;
1218
1219         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1220                 goto process_auth_encryption_ccm_err;
1221
1222         return 0;
1223
1224 process_auth_encryption_ccm_err:
1225         OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed");
1226         return -EINVAL;
1227 }
1228
1229 /** Process AES-GCM decrypt algorithm */
1230 static int
1231 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1232                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1233                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1234 {
1235         int len = 0, unused = 0;
1236         uint8_t empty[] = {};
1237
1238         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1239                 goto process_auth_decryption_gcm_err;
1240
1241         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1242                 goto process_auth_decryption_gcm_err;
1243
1244         if (aadlen > 0)
1245                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1246                         goto process_auth_decryption_gcm_err;
1247
1248         if (srclen > 0)
1249                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1250                                 srclen, ctx, 0))
1251                         goto process_auth_decryption_gcm_err;
1252
1253         /* Workaround open ssl bug in version less then 1.0.1f */
1254         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1255                 goto process_auth_decryption_gcm_err;
1256
1257         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1258                 return -EFAULT;
1259
1260         return 0;
1261
1262 process_auth_decryption_gcm_err:
1263         OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed");
1264         return -EINVAL;
1265 }
1266
1267 /** Process AES-CCM decrypt algorithm */
1268 static int
1269 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1270                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1271                 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1272                 EVP_CIPHER_CTX *ctx)
1273 {
1274         int len = 0;
1275
1276         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1277                 goto process_auth_decryption_ccm_err;
1278
1279         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1280                 goto process_auth_decryption_ccm_err;
1281
1282         if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1283                 goto process_auth_decryption_ccm_err;
1284
1285         if (aadlen > 0)
1286                 /*
1287                  * For AES-CCM, the actual AAD is placed
1288                  * 18 bytes after the start of the AAD field,
1289                  * according to the API.
1290                  */
1291                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1292                         goto process_auth_decryption_ccm_err;
1293
1294         if (srclen >= 0)
1295                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1296                                 srclen, ctx, 0))
1297                         return -EFAULT;
1298
1299         return 0;
1300
1301 process_auth_decryption_ccm_err:
1302         OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed");
1303         return -EINVAL;
1304 }
1305
1306 /** Process standard openssl auth algorithms */
1307 static int
1308 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1309                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1310                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1311 {
1312         size_t dstlen;
1313         struct rte_mbuf *m;
1314         int l, n = srclen;
1315         uint8_t *src;
1316
1317         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1318                         m = m->next)
1319                 offset -= rte_pktmbuf_data_len(m);
1320
1321         if (m == 0)
1322                 goto process_auth_err;
1323
1324         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1325                 goto process_auth_err;
1326
1327         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1328
1329         l = rte_pktmbuf_data_len(m) - offset;
1330         if (srclen <= l) {
1331                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1332                         goto process_auth_err;
1333                 goto process_auth_final;
1334         }
1335
1336         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1337                 goto process_auth_err;
1338
1339         n -= l;
1340
1341         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1342                 src = rte_pktmbuf_mtod(m, uint8_t *);
1343                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1344                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1345                         goto process_auth_err;
1346                 n -= l;
1347         }
1348
1349 process_auth_final:
1350         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1351                 goto process_auth_err;
1352         return 0;
1353
1354 process_auth_err:
1355         OPENSSL_LOG(ERR, "Process openssl auth failed");
1356         return -EINVAL;
1357 }
1358
1359 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1360 /** Process standard openssl auth algorithms with hmac */
1361 static int
1362 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1363                 int srclen, EVP_MAC_CTX *ctx)
1364 {
1365         size_t dstlen;
1366         struct rte_mbuf *m;
1367         int l, n = srclen;
1368         uint8_t *src;
1369
1370         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1371                         m = m->next)
1372                 offset -= rte_pktmbuf_data_len(m);
1373
1374         if (m == 0)
1375                 goto process_auth_err;
1376
1377         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1378
1379         l = rte_pktmbuf_data_len(m) - offset;
1380         if (srclen <= l) {
1381                 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1)
1382                         goto process_auth_err;
1383                 goto process_auth_final;
1384         }
1385
1386         if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1387                 goto process_auth_err;
1388
1389         n -= l;
1390
1391         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1392                 src = rte_pktmbuf_mtod(m, uint8_t *);
1393                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1394                 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1395                         goto process_auth_err;
1396                 n -= l;
1397         }
1398
1399 process_auth_final:
1400         if (EVP_MAC_final(ctx, dst, &dstlen, sizeof(dst)) != 1)
1401                 goto process_auth_err;
1402
1403         EVP_MAC_CTX_free(ctx);
1404         return 0;
1405
1406 process_auth_err:
1407         EVP_MAC_CTX_free(ctx);
1408         OPENSSL_LOG(ERR, "Process openssl auth failed");
1409         return -EINVAL;
1410 }
1411 # else
1412 /** Process standard openssl auth algorithms with hmac */
1413 static int
1414 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1415                 int srclen, HMAC_CTX *ctx)
1416 {
1417         unsigned int dstlen;
1418         struct rte_mbuf *m;
1419         int l, n = srclen;
1420         uint8_t *src;
1421
1422         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1423                         m = m->next)
1424                 offset -= rte_pktmbuf_data_len(m);
1425
1426         if (m == 0)
1427                 goto process_auth_err;
1428
1429         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1430
1431         l = rte_pktmbuf_data_len(m) - offset;
1432         if (srclen <= l) {
1433                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1434                         goto process_auth_err;
1435                 goto process_auth_final;
1436         }
1437
1438         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1439                 goto process_auth_err;
1440
1441         n -= l;
1442
1443         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1444                 src = rte_pktmbuf_mtod(m, uint8_t *);
1445                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1446                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1447                         goto process_auth_err;
1448                 n -= l;
1449         }
1450
1451 process_auth_final:
1452         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1453                 goto process_auth_err;
1454
1455         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1456                 goto process_auth_err;
1457
1458         return 0;
1459
1460 process_auth_err:
1461         OPENSSL_LOG(ERR, "Process openssl auth failed");
1462         return -EINVAL;
1463 }
1464 # endif
1465 /*----------------------------------------------------------------------------*/
1466
1467 /** Process auth/cipher combined operation */
1468 static void
1469 process_openssl_combined_op
1470                 (struct rte_crypto_op *op, struct openssl_session *sess,
1471                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1472 {
1473         /* cipher */
1474         uint8_t *dst = NULL, *iv, *tag, *aad;
1475         int srclen, aadlen, status = -1;
1476         uint32_t offset;
1477         uint8_t taglen;
1478
1479         /*
1480          * Segmented destination buffer is not supported for
1481          * encryption/decryption
1482          */
1483         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1484                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1485                 return;
1486         }
1487
1488         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1489                         sess->iv.offset);
1490         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1491                 srclen = 0;
1492                 offset = op->sym->auth.data.offset;
1493                 aadlen = op->sym->auth.data.length;
1494                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1495                                 op->sym->auth.data.offset);
1496                 tag = op->sym->auth.digest.data;
1497                 if (tag == NULL)
1498                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1499                                 offset + aadlen);
1500         } else {
1501                 srclen = op->sym->aead.data.length;
1502                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1503                                 op->sym->aead.data.offset);
1504                 offset = op->sym->aead.data.offset;
1505                 aad = op->sym->aead.aad.data;
1506                 aadlen = sess->auth.aad_length;
1507                 tag = op->sym->aead.digest.data;
1508                 if (tag == NULL)
1509                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1510                                 offset + srclen);
1511         }
1512
1513         taglen = sess->auth.digest_length;
1514
1515         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1516                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1517                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1518                         status = process_openssl_auth_encryption_gcm(
1519                                         mbuf_src, offset, srclen,
1520                                         aad, aadlen, iv,
1521                                         dst, tag, sess->cipher.ctx);
1522                 else
1523                         status = process_openssl_auth_encryption_ccm(
1524                                         mbuf_src, offset, srclen,
1525                                         aad, aadlen, iv,
1526                                         dst, tag, taglen, sess->cipher.ctx);
1527
1528         } else {
1529                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1530                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1531                         status = process_openssl_auth_decryption_gcm(
1532                                         mbuf_src, offset, srclen,
1533                                         aad, aadlen, iv,
1534                                         dst, tag, sess->cipher.ctx);
1535                 else
1536                         status = process_openssl_auth_decryption_ccm(
1537                                         mbuf_src, offset, srclen,
1538                                         aad, aadlen, iv,
1539                                         dst, tag, taglen, sess->cipher.ctx);
1540         }
1541
1542         if (status != 0) {
1543                 if (status == (-EFAULT) &&
1544                                 sess->auth.operation ==
1545                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1546                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1547                 else
1548                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1549         }
1550 }
1551
1552 /** Process cipher operation */
1553 static void
1554 process_openssl_cipher_op
1555                 (struct rte_crypto_op *op, struct openssl_session *sess,
1556                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1557 {
1558         uint8_t *dst, *iv;
1559         int srclen, status;
1560         uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0;
1561         EVP_CIPHER_CTX *ctx_copy;
1562
1563         /*
1564          * Segmented OOP destination buffer is not supported for encryption/
1565          * decryption. In case of des3ctr, even inplace segmented buffers are
1566          * not supported.
1567          */
1568         if (!rte_pktmbuf_is_contiguous(mbuf_dst) &&
1569                         (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) {
1570                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1571                 return;
1572         }
1573
1574         srclen = op->sym->cipher.data.length;
1575         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1576                         op->sym->cipher.data.offset);
1577
1578         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1579                         sess->iv.offset);
1580         ctx_copy = EVP_CIPHER_CTX_new();
1581         EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx);
1582
1583         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1584                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1585                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1586                                         op->sym->cipher.data.offset, iv,
1587                                         srclen, ctx_copy, inplace);
1588                 else
1589                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1590                                         op->sym->cipher.data.offset, iv,
1591                                         srclen, ctx_copy, inplace);
1592         else
1593                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1594                                 op->sym->cipher.data.offset, iv,
1595                                 sess->cipher.key.data, srclen,
1596                                 ctx_copy);
1597
1598         EVP_CIPHER_CTX_free(ctx_copy);
1599         if (status != 0)
1600                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1601 }
1602
1603 /** Process cipher operation */
1604 static void
1605 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1606                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1607                 struct rte_mbuf *mbuf_dst)
1608 {
1609         uint8_t *src, *dst, *iv;
1610         uint8_t block_size, last_block_len;
1611         int srclen, status = 0;
1612
1613         srclen = op->sym->cipher.data.length;
1614         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1615                         op->sym->cipher.data.offset);
1616         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1617                         op->sym->cipher.data.offset);
1618
1619         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1620                         sess->iv.offset);
1621
1622         block_size = DES_BLOCK_SIZE;
1623
1624         last_block_len = srclen % block_size;
1625         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1626                 /* Encrypt only with ECB mode XOR IV */
1627                 if (srclen < block_size) {
1628                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1629                                         iv, srclen,
1630                                         sess->cipher.bpi_ctx);
1631                 } else {
1632                         srclen -= last_block_len;
1633                         /* Encrypt with the block aligned stream with CBC mode */
1634                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1635                                         op->sym->cipher.data.offset, iv,
1636                                         srclen, sess->cipher.ctx, 0);
1637                         if (last_block_len) {
1638                                 /* Point at last block */
1639                                 dst += srclen;
1640                                 /*
1641                                  * IV is the last encrypted block from
1642                                  * the previous operation
1643                                  */
1644                                 iv = dst - block_size;
1645                                 src += srclen;
1646                                 srclen = last_block_len;
1647                                 /* Encrypt the last frame with ECB mode */
1648                                 status |= process_openssl_cipher_bpi_encrypt(src,
1649                                                 dst, iv,
1650                                                 srclen, sess->cipher.bpi_ctx);
1651                         }
1652                 }
1653         } else {
1654                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1655                 if (srclen < block_size) {
1656                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1657                                         iv,
1658                                         srclen,
1659                                         sess->cipher.bpi_ctx);
1660                 } else {
1661                         if (last_block_len) {
1662                                 /* Point at last block */
1663                                 dst += srclen - last_block_len;
1664                                 src += srclen - last_block_len;
1665                                 /*
1666                                  * IV is the last full block
1667                                  */
1668                                 iv = src - block_size;
1669                                 /*
1670                                  * Decrypt the last frame with ECB mode
1671                                  * (encrypt, as it is the same operation)
1672                                  */
1673                                 status = process_openssl_cipher_bpi_encrypt(src,
1674                                                 dst, iv,
1675                                                 last_block_len, sess->cipher.bpi_ctx);
1676                                 /* Prepare parameters for CBC mode op */
1677                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1678                                                 sess->iv.offset);
1679                                 dst += last_block_len - srclen;
1680                                 srclen -= last_block_len;
1681                         }
1682
1683                         /* Decrypt with CBC mode */
1684                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1685                                         op->sym->cipher.data.offset, iv,
1686                                         srclen, sess->cipher.ctx, 0);
1687                 }
1688         }
1689
1690         if (status != 0)
1691                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1692 }
1693
1694 /** Process auth operation */
1695 static void
1696 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1697                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1698                 struct rte_mbuf *mbuf_dst)
1699 {
1700         uint8_t *dst;
1701         int srclen, status;
1702         EVP_MD_CTX *ctx_a;
1703 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1704         EVP_MAC_CTX *ctx_h;
1705         EVP_MAC *mac;
1706 # else
1707         HMAC_CTX *ctx_h;
1708 # endif
1709
1710         srclen = op->sym->auth.data.length;
1711
1712         dst = qp->temp_digest;
1713
1714         switch (sess->auth.mode) {
1715         case OPENSSL_AUTH_AS_AUTH:
1716                 ctx_a = EVP_MD_CTX_create();
1717                 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx);
1718                 status = process_openssl_auth(mbuf_src, dst,
1719                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1720                                 ctx_a, sess->auth.auth.evp_algo);
1721                 EVP_MD_CTX_destroy(ctx_a);
1722                 break;
1723         case OPENSSL_AUTH_AS_HMAC:
1724 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1725                 mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
1726                 ctx_h = EVP_MAC_CTX_new(mac);
1727                 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx);
1728                 EVP_MAC_free(mac);
1729                 status = process_openssl_auth_hmac(mbuf_src, dst,
1730                                 op->sym->auth.data.offset, srclen,
1731                                 ctx_h);
1732 # else
1733                 ctx_h = HMAC_CTX_new();
1734                 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx);
1735                 status = process_openssl_auth_hmac(mbuf_src, dst,
1736                                 op->sym->auth.data.offset, srclen,
1737                                 ctx_h);
1738                 HMAC_CTX_free(ctx_h);
1739 # endif
1740                 break;
1741         default:
1742                 status = -1;
1743                 break;
1744         }
1745
1746         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1747                 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data,
1748                                 sess->auth.digest_length) != 0) {
1749                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1750                 }
1751         } else {
1752                 uint8_t *auth_dst;
1753
1754                 auth_dst = op->sym->auth.digest.data;
1755                 if (auth_dst == NULL)
1756                         auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1757                                         op->sym->auth.data.offset +
1758                                         op->sym->auth.data.length);
1759                 memcpy(auth_dst, dst, sess->auth.digest_length);
1760         }
1761
1762         if (status != 0)
1763                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1764 }
1765
1766 /* process dsa sign operation */
1767 static int
1768 process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
1769                 struct openssl_asym_session *sess)
1770 {
1771         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1772         DSA *dsa = sess->u.s.dsa;
1773         DSA_SIG *sign = NULL;
1774
1775         sign = DSA_do_sign(op->message.data,
1776                         op->message.length,
1777                         dsa);
1778
1779         if (sign == NULL) {
1780                 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
1781                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1782         } else {
1783                 const BIGNUM *r = NULL, *s = NULL;
1784                 get_dsa_sign(sign, &r, &s);
1785
1786                 op->r.length = BN_bn2bin(r, op->r.data);
1787                 op->s.length = BN_bn2bin(s, op->s.data);
1788                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1789         }
1790
1791         DSA_SIG_free(sign);
1792
1793         return 0;
1794 }
1795
1796 /* process dsa verify operation */
1797 static int
1798 process_openssl_dsa_verify_op(struct rte_crypto_op *cop,
1799                 struct openssl_asym_session *sess)
1800 {
1801         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1802         DSA *dsa = sess->u.s.dsa;
1803         int ret;
1804         DSA_SIG *sign = DSA_SIG_new();
1805         BIGNUM *r = NULL, *s = NULL;
1806         BIGNUM *pub_key = NULL;
1807
1808         if (sign == NULL) {
1809                 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
1810                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1811                 return -1;
1812         }
1813
1814         r = BN_bin2bn(op->r.data,
1815                         op->r.length,
1816                         r);
1817         s = BN_bin2bn(op->s.data,
1818                         op->s.length,
1819                         s);
1820         pub_key = BN_bin2bn(op->y.data,
1821                         op->y.length,
1822                         pub_key);
1823         if (!r || !s || !pub_key) {
1824                 BN_free(r);
1825                 BN_free(s);
1826                 BN_free(pub_key);
1827
1828                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1829                 return -1;
1830         }
1831         set_dsa_sign(sign, r, s);
1832         set_dsa_pub_key(dsa, pub_key);
1833
1834         ret = DSA_do_verify(op->message.data,
1835                         op->message.length,
1836                         sign,
1837                         dsa);
1838
1839         if (ret != 1)
1840                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1841         else
1842                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1843
1844         DSA_SIG_free(sign);
1845
1846         return 0;
1847 }
1848
1849 /* process dh operation */
1850 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
1851 static int
1852 process_openssl_dh_op_evp(struct rte_crypto_op *cop,
1853                 struct openssl_asym_session *sess)
1854 {
1855         struct rte_crypto_dh_op_param *op = &cop->asym->dh;
1856         OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld;
1857         OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer;
1858         OSSL_PARAM *params = NULL;
1859         EVP_PKEY *dhpkey = NULL;
1860         EVP_PKEY *peerkey = NULL;
1861         BIGNUM *priv_key = NULL;
1862         BIGNUM *pub_key = NULL;
1863         int ret = -1;
1864
1865         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1866         EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
1867         if (dh_ctx == NULL || param_bld == NULL)
1868                 return ret;
1869
1870         if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
1871                 OSSL_PARAM *params_peer = NULL;
1872
1873                 if (!param_bld_peer)
1874                         return ret;
1875
1876                 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length,
1877                                         pub_key);
1878                 if (pub_key == NULL) {
1879                         OSSL_PARAM_BLD_free(param_bld_peer);
1880                         return ret;
1881                 }
1882
1883                 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY,
1884                                 pub_key)) {
1885                         OPENSSL_LOG(ERR, "Failed to set public key\n");
1886                         OSSL_PARAM_BLD_free(param_bld_peer);
1887                         BN_free(pub_key);
1888                         return ret;
1889                 }
1890
1891                 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer);
1892                 if (!params_peer) {
1893                         OSSL_PARAM_BLD_free(param_bld_peer);
1894                         BN_free(pub_key);
1895                         return ret;
1896                 }
1897
1898                 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
1899                 if (EVP_PKEY_keygen_init(peer_ctx) != 1) {
1900                         OSSL_PARAM_free(params_peer);
1901                         BN_free(pub_key);
1902                         return ret;
1903                 }
1904
1905                 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) {
1906                         EVP_PKEY_CTX_free(peer_ctx);
1907                         OSSL_PARAM_free(params_peer);
1908                         BN_free(pub_key);
1909                         return ret;
1910                 }
1911
1912                 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) {
1913                         EVP_PKEY_CTX_free(peer_ctx);
1914                         OSSL_PARAM_free(params_peer);
1915                         BN_free(pub_key);
1916                         return ret;
1917                 }
1918
1919                 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length,
1920                                         priv_key);
1921                 if (priv_key == NULL) {
1922                         EVP_PKEY_CTX_free(peer_ctx);
1923                         OSSL_PARAM_free(params_peer);
1924                         BN_free(pub_key);
1925                         return ret;
1926                 }
1927
1928                 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY,
1929                                 priv_key)) {
1930                         OPENSSL_LOG(ERR, "Failed to set private key\n");
1931                         EVP_PKEY_CTX_free(peer_ctx);
1932                         OSSL_PARAM_free(params_peer);
1933                         BN_free(pub_key);
1934                         BN_free(priv_key);
1935                         return ret;
1936                 }
1937
1938                 OSSL_PARAM_free(params_peer);
1939                 EVP_PKEY_CTX_free(peer_ctx);
1940         }
1941
1942         params = OSSL_PARAM_BLD_to_param(param_bld);
1943         if (!params)
1944                 goto err_dh;
1945
1946         if (EVP_PKEY_keygen_init(dh_ctx) != 1)
1947                 goto err_dh;
1948
1949         if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1)
1950                 goto err_dh;
1951
1952         if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1)
1953                 goto err_dh;
1954
1955         if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
1956                 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__);
1957                 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key))
1958                         goto err_dh;
1959                                 /* output public key */
1960                 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data);
1961         }
1962
1963         if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
1964
1965                 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__);
1966                 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key))
1967                         goto err_dh;
1968
1969                 /* provide generated private key back to user */
1970                 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data);
1971         }
1972
1973         if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
1974                 size_t skey_len;
1975                 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL);
1976                 if (!sc_ctx)
1977                         goto err_dh;
1978
1979                 if (EVP_PKEY_derive_init(sc_ctx) <= 0) {
1980                         EVP_PKEY_CTX_free(sc_ctx);
1981                         goto err_dh;
1982                 }
1983
1984                 if (!peerkey) {
1985                         EVP_PKEY_CTX_free(sc_ctx);
1986                         goto err_dh;
1987                 }
1988
1989                 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) {
1990                         EVP_PKEY_CTX_free(sc_ctx);
1991                         goto err_dh;
1992                 }
1993
1994                 /* Determine buffer length */
1995                 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) {
1996                         EVP_PKEY_CTX_free(sc_ctx);
1997                         goto err_dh;
1998                 }
1999
2000                 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) {
2001                         EVP_PKEY_CTX_free(sc_ctx);
2002                         goto err_dh;
2003                 }
2004
2005                 op->shared_secret.length = skey_len;
2006                 EVP_PKEY_CTX_free(sc_ctx);
2007         }
2008
2009         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2010         ret = 0;
2011
2012  err_dh:
2013         if (pub_key)
2014                 BN_free(pub_key);
2015         if (priv_key)
2016                 BN_free(priv_key);
2017         if (params)
2018                 OSSL_PARAM_free(params);
2019         if (dhpkey)
2020                 EVP_PKEY_free(dhpkey);
2021         if (peerkey)
2022                 EVP_PKEY_free(peerkey);
2023
2024         EVP_PKEY_CTX_free(dh_ctx);
2025
2026         return ret;
2027 }
2028 #else
2029 static int
2030 process_openssl_dh_op(struct rte_crypto_op *cop,
2031                 struct openssl_asym_session *sess)
2032 {
2033         struct rte_crypto_dh_op_param *op = &cop->asym->dh;
2034         struct rte_crypto_asym_op *asym_op = cop->asym;
2035         DH *dh_key = sess->u.dh.dh_key;
2036         BIGNUM *priv_key = NULL;
2037         int ret = 0;
2038
2039         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2040                 /* compute shared secret using peer public key
2041                  * and current private key
2042                  * shared secret = peer_key ^ priv_key mod p
2043                  */
2044                 BIGNUM *peer_key = NULL;
2045
2046                 /* copy private key and peer key and compute shared secret */
2047                 peer_key = BN_bin2bn(op->pub_key.data,
2048                                 op->pub_key.length,
2049                                 peer_key);
2050                 if (peer_key == NULL) {
2051                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2052                         return -1;
2053                 }
2054                 priv_key = BN_bin2bn(op->priv_key.data,
2055                                 op->priv_key.length,
2056                                 priv_key);
2057                 if (priv_key == NULL) {
2058                         BN_free(peer_key);
2059                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2060                         return -1;
2061                 }
2062                 ret = set_dh_priv_key(dh_key, priv_key);
2063                 if (ret) {
2064                         OPENSSL_LOG(ERR, "Failed to set private key\n");
2065                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2066                         BN_free(peer_key);
2067                         BN_free(priv_key);
2068                         return 0;
2069                 }
2070
2071                 ret = DH_compute_key(
2072                                 op->shared_secret.data,
2073                                 peer_key, dh_key);
2074                 if (ret < 0) {
2075                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2076                         BN_free(peer_key);
2077                         /* priv key is already loaded into dh,
2078                          * let's not free that directly here.
2079                          * DH_free() will auto free it later.
2080                          */
2081                         return 0;
2082                 }
2083                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2084                 op->shared_secret.length = ret;
2085                 BN_free(peer_key);
2086                 return 0;
2087         }
2088
2089         /*
2090          * other options are public and private key generations.
2091          *
2092          * if user provides private key,
2093          * then first set DH with user provided private key
2094          */
2095         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE &&
2096                         op->priv_key.length) {
2097                 /* generate public key using user-provided private key
2098                  * pub_key = g ^ priv_key mod p
2099                  */
2100
2101                 /* load private key into DH */
2102                 priv_key = BN_bin2bn(op->priv_key.data,
2103                                 op->priv_key.length,
2104                                 priv_key);
2105                 if (priv_key == NULL) {
2106                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2107                         return -1;
2108                 }
2109                 ret = set_dh_priv_key(dh_key, priv_key);
2110                 if (ret) {
2111                         OPENSSL_LOG(ERR, "Failed to set private key\n");
2112                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2113                         BN_free(priv_key);
2114                         return 0;
2115                 }
2116         }
2117
2118         /* generate public and private key pair.
2119          *
2120          * if private key already set, generates only public key.
2121          *
2122          * if private key is not already set, then set it to random value
2123          * and update internal private key.
2124          */
2125         if (!DH_generate_key(dh_key)) {
2126                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2127                 return 0;
2128         }
2129
2130         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
2131                 const BIGNUM *pub_key = NULL;
2132
2133                 OPENSSL_LOG(DEBUG, "%s:%d update public key\n",
2134                                 __func__, __LINE__);
2135
2136                 /* get the generated keys */
2137                 get_dh_pub_key(dh_key, &pub_key);
2138
2139                 /* output public key */
2140                 op->pub_key.length = BN_bn2bin(pub_key,
2141                                 op->pub_key.data);
2142         }
2143
2144         if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
2145                 const BIGNUM *priv_key = NULL;
2146
2147                 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n",
2148                                 __func__, __LINE__);
2149
2150                 /* get the generated keys */
2151                 get_dh_priv_key(dh_key, &priv_key);
2152
2153                 /* provide generated private key back to user */
2154                 op->priv_key.length = BN_bn2bin(priv_key,
2155                                 op->priv_key.data);
2156         }
2157
2158         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2159
2160         return 0;
2161 }
2162 #endif
2163
2164 /* process modinv operation */
2165 static int
2166 process_openssl_modinv_op(struct rte_crypto_op *cop,
2167                 struct openssl_asym_session *sess)
2168 {
2169         struct rte_crypto_asym_op *op = cop->asym;
2170         BIGNUM *base = BN_CTX_get(sess->u.m.ctx);
2171         BIGNUM *res = BN_CTX_get(sess->u.m.ctx);
2172
2173         if (unlikely(base == NULL || res == NULL)) {
2174                 BN_free(base);
2175                 BN_free(res);
2176                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2177                 return -1;
2178         }
2179
2180         base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
2181                         op->modinv.base.length, base);
2182
2183         if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) {
2184                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2185                 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data);
2186         } else {
2187                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2188         }
2189
2190         BN_clear(res);
2191         BN_clear(base);
2192
2193         return 0;
2194 }
2195
2196 /* process modexp operation */
2197 static int
2198 process_openssl_modexp_op(struct rte_crypto_op *cop,
2199                 struct openssl_asym_session *sess)
2200 {
2201         struct rte_crypto_asym_op *op = cop->asym;
2202         BIGNUM *base = BN_CTX_get(sess->u.e.ctx);
2203         BIGNUM *res = BN_CTX_get(sess->u.e.ctx);
2204
2205         if (unlikely(base == NULL || res == NULL)) {
2206                 BN_free(base);
2207                 BN_free(res);
2208                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2209                 return -1;
2210         }
2211
2212         base = BN_bin2bn((const unsigned char *)op->modex.base.data,
2213                         op->modex.base.length, base);
2214
2215         if (BN_mod_exp(res, base, sess->u.e.exp,
2216                                 sess->u.e.mod, sess->u.e.ctx)) {
2217                 op->modex.result.length = BN_bn2bin(res, op->modex.result.data);
2218                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2219         } else {
2220                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2221         }
2222
2223         BN_clear(res);
2224         BN_clear(base);
2225
2226         return 0;
2227 }
2228
2229 /* process rsa operations */
2230 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2231 static int
2232 process_openssl_rsa_op_evp(struct rte_crypto_op *cop,
2233                 struct openssl_asym_session *sess)
2234 {
2235         struct rte_crypto_asym_op *op = cop->asym;
2236         uint32_t pad = (op->rsa.padding.type);
2237         uint8_t *tmp;
2238         size_t outlen = 0;
2239         int ret = -1;
2240
2241         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2242         EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx;
2243         if (!rsa_ctx)
2244                 return ret;
2245
2246         switch (pad) {
2247         case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
2248                 pad = RSA_PKCS1_PADDING;
2249                 break;
2250         case RTE_CRYPTO_RSA_PADDING_NONE:
2251                 pad = RSA_NO_PADDING;
2252                 break;
2253         default:
2254                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2255                 OPENSSL_LOG(ERR,
2256                                 "rsa pad type not supported %d\n", pad);
2257                 return ret;
2258         }
2259
2260         switch (op->rsa.op_type) {
2261         case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2262                 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1)
2263                         goto err_rsa;
2264
2265                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2266                         goto err_rsa;
2267
2268                 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen,
2269                                 op->rsa.message.data,
2270                                 op->rsa.message.length) <= 0)
2271                         goto err_rsa;
2272
2273                 if (outlen <= 0)
2274                         goto err_rsa;
2275
2276                 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen,
2277                                 op->rsa.message.data,
2278                                 op->rsa.message.length) <= 0)
2279                         goto err_rsa;
2280                 op->rsa.cipher.length = outlen;
2281
2282                 OPENSSL_LOG(DEBUG,
2283                                 "length of encrypted text %zu\n", outlen);
2284                 break;
2285
2286         case RTE_CRYPTO_ASYM_OP_DECRYPT:
2287                 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1)
2288                         goto err_rsa;
2289
2290                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2291                         goto err_rsa;
2292
2293                 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen,
2294                                 op->rsa.cipher.data,
2295                                 op->rsa.cipher.length) <= 0)
2296                         goto err_rsa;
2297
2298                 if (outlen <= 0)
2299                         goto err_rsa;
2300
2301                 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen,
2302                                 op->rsa.cipher.data,
2303                                 op->rsa.cipher.length) <= 0)
2304                         goto err_rsa;
2305                 op->rsa.message.length = outlen;
2306
2307                 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen);
2308                 break;
2309
2310         case RTE_CRYPTO_ASYM_OP_SIGN:
2311                 if (EVP_PKEY_sign_init(rsa_ctx) <= 0)
2312                         goto err_rsa;
2313
2314                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2315                         goto err_rsa;
2316
2317                 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen,
2318                                 op->rsa.message.data,
2319                                 op->rsa.message.length) <= 0)
2320                         goto err_rsa;
2321                 op->rsa.sign.length = outlen;
2322                 break;
2323
2324         case RTE_CRYPTO_ASYM_OP_VERIFY:
2325                 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
2326                 if (tmp == NULL) {
2327                         OPENSSL_LOG(ERR, "Memory allocation failed");
2328                         goto err_rsa;
2329                 }
2330
2331                 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) {
2332                         rte_free(tmp);
2333                         goto err_rsa;
2334                 }
2335
2336                 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) {
2337                         rte_free(tmp);
2338                         goto err_rsa;
2339                 }
2340
2341                 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen,
2342                                 op->rsa.sign.data,
2343                                 op->rsa.sign.length) <= 0) {
2344                         rte_free(tmp);
2345                         goto err_rsa;
2346                 }
2347
2348                 OPENSSL_LOG(DEBUG,
2349                                 "Length of public_decrypt %zu "
2350                                 "length of message %zd\n",
2351                                 outlen, op->rsa.message.length);
2352                 if (CRYPTO_memcmp(tmp, op->rsa.message.data,
2353                                 op->rsa.message.length)) {
2354                         OPENSSL_LOG(ERR, "RSA sign Verification failed");
2355                 }
2356                 rte_free(tmp);
2357                 break;
2358
2359         default:
2360                 /* allow ops with invalid args to be pushed to
2361                  * completion queue
2362                  */
2363                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2364                 goto err_rsa;
2365         }
2366
2367         ret = 0;
2368         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2369 err_rsa:
2370         return ret;
2371
2372 }
2373 #else
2374 static int
2375 process_openssl_rsa_op(struct rte_crypto_op *cop,
2376                 struct openssl_asym_session *sess)
2377 {
2378         int ret = 0;
2379         struct rte_crypto_asym_op *op = cop->asym;
2380         RSA *rsa = sess->u.r.rsa;
2381         uint32_t pad = (op->rsa.padding.type);
2382         uint8_t *tmp;
2383
2384         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2385
2386         switch (pad) {
2387         case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
2388                 pad = RSA_PKCS1_PADDING;
2389                 break;
2390         case RTE_CRYPTO_RSA_PADDING_NONE:
2391                 pad = RSA_NO_PADDING;
2392                 break;
2393         default:
2394                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2395                 OPENSSL_LOG(ERR,
2396                                 "rsa pad type not supported %d\n", pad);
2397                 return 0;
2398         }
2399
2400         switch (op->rsa.op_type) {
2401         case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2402                 ret = RSA_public_encrypt(op->rsa.message.length,
2403                                 op->rsa.message.data,
2404                                 op->rsa.cipher.data,
2405                                 rsa,
2406                                 pad);
2407
2408                 if (ret > 0)
2409                         op->rsa.cipher.length = ret;
2410                 OPENSSL_LOG(DEBUG,
2411                                 "length of encrypted text %d\n", ret);
2412                 break;
2413
2414         case RTE_CRYPTO_ASYM_OP_DECRYPT:
2415                 ret = RSA_private_decrypt(op->rsa.cipher.length,
2416                                 op->rsa.cipher.data,
2417                                 op->rsa.message.data,
2418                                 rsa,
2419                                 pad);
2420                 if (ret > 0)
2421                         op->rsa.message.length = ret;
2422                 break;
2423
2424         case RTE_CRYPTO_ASYM_OP_SIGN:
2425                 ret = RSA_private_encrypt(op->rsa.message.length,
2426                                 op->rsa.message.data,
2427                                 op->rsa.sign.data,
2428                                 rsa,
2429                                 pad);
2430                 if (ret > 0)
2431                         op->rsa.sign.length = ret;
2432                 break;
2433
2434         case RTE_CRYPTO_ASYM_OP_VERIFY:
2435                 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
2436                 if (tmp == NULL) {
2437                         OPENSSL_LOG(ERR, "Memory allocation failed");
2438                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2439                         break;
2440                 }
2441                 ret = RSA_public_decrypt(op->rsa.sign.length,
2442                                 op->rsa.sign.data,
2443                                 tmp,
2444                                 rsa,
2445                                 pad);
2446
2447                 OPENSSL_LOG(DEBUG,
2448                                 "Length of public_decrypt %d "
2449                                 "length of message %zd\n",
2450                                 ret, op->rsa.message.length);
2451                 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data,
2452                                 op->rsa.message.length))) {
2453                         OPENSSL_LOG(ERR, "RSA sign Verification failed");
2454                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2455                 }
2456                 rte_free(tmp);
2457                 break;
2458
2459         default:
2460                 /* allow ops with invalid args to be pushed to
2461                  * completion queue
2462                  */
2463                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2464                 break;
2465         }
2466
2467         if (ret < 0)
2468                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2469
2470         return 0;
2471 }
2472 #endif
2473
2474 static int
2475 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2476                 struct openssl_asym_session *sess)
2477 {
2478         int retval = 0;
2479
2480         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2481
2482         switch (sess->xfrm_type) {
2483         case RTE_CRYPTO_ASYM_XFORM_RSA:
2484 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2485                 retval = process_openssl_rsa_op_evp(op, sess);
2486 # else
2487                 retval = process_openssl_rsa_op(op, sess);
2488 #endif
2489                 break;
2490         case RTE_CRYPTO_ASYM_XFORM_MODEX:
2491                 retval = process_openssl_modexp_op(op, sess);
2492                 break;
2493         case RTE_CRYPTO_ASYM_XFORM_MODINV:
2494                 retval = process_openssl_modinv_op(op, sess);
2495                 break;
2496         case RTE_CRYPTO_ASYM_XFORM_DH:
2497 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2498                 retval = process_openssl_dh_op_evp(op, sess);
2499 # else
2500                 retval = process_openssl_dh_op(op, sess);
2501 #endif
2502                 break;
2503         case RTE_CRYPTO_ASYM_XFORM_DSA:
2504                 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
2505                         retval = process_openssl_dsa_sign_op(op, sess);
2506                 else if (op->asym->dsa.op_type ==
2507                                 RTE_CRYPTO_ASYM_OP_VERIFY)
2508                         retval =
2509                                 process_openssl_dsa_verify_op(op, sess);
2510                 else
2511                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2512                 break;
2513         default:
2514                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2515                 break;
2516         }
2517         if (!retval) {
2518                 /* op processed so push to completion queue as processed */
2519                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2520                 if (retval)
2521                         /* return error if failed to put in completion queue */
2522                         retval = -1;
2523         }
2524
2525         return retval;
2526 }
2527
2528 static void
2529 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst,
2530                 struct rte_crypto_op *op)
2531 {
2532         uint8_t *p_src, *p_dst;
2533
2534         p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
2535         p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
2536
2537         /**
2538          * Copy the content between cipher offset and auth offset
2539          * for generating correct digest.
2540          */
2541         if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
2542                 memcpy(p_dst + op->sym->auth.data.offset,
2543                                 p_src + op->sym->auth.data.offset,
2544                                 op->sym->cipher.data.offset -
2545                                 op->sym->auth.data.offset);
2546 }
2547
2548 /** Process crypto operation for mbuf */
2549 static int
2550 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2551                 struct openssl_session *sess)
2552 {
2553         struct rte_mbuf *msrc, *mdst;
2554         int retval;
2555
2556         msrc = op->sym->m_src;
2557         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
2558
2559         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2560
2561         switch (sess->chain_order) {
2562         case OPENSSL_CHAIN_ONLY_CIPHER:
2563                 process_openssl_cipher_op(op, sess, msrc, mdst);
2564                 break;
2565         case OPENSSL_CHAIN_ONLY_AUTH:
2566                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2567                 break;
2568         case OPENSSL_CHAIN_CIPHER_AUTH:
2569                 process_openssl_cipher_op(op, sess, msrc, mdst);
2570                 /* OOP */
2571                 if (msrc != mdst)
2572                         copy_plaintext(msrc, mdst, op);
2573                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
2574                 break;
2575         case OPENSSL_CHAIN_AUTH_CIPHER:
2576                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2577                 process_openssl_cipher_op(op, sess, msrc, mdst);
2578                 break;
2579         case OPENSSL_CHAIN_COMBINED:
2580                 process_openssl_combined_op(op, sess, msrc, mdst);
2581                 break;
2582         case OPENSSL_CHAIN_CIPHER_BPI:
2583                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
2584                 break;
2585         default:
2586                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
2587                 break;
2588         }
2589
2590         /* Free session if a session-less crypto op */
2591         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
2592                 openssl_reset_session(sess);
2593                 memset(sess, 0, sizeof(struct openssl_session));
2594                 memset(op->sym->session, 0,
2595                         rte_cryptodev_sym_get_existing_header_session_size(
2596                                 op->sym->session));
2597                 rte_mempool_put(qp->sess_mp_priv, sess);
2598                 rte_mempool_put(qp->sess_mp, op->sym->session);
2599                 op->sym->session = NULL;
2600         }
2601
2602         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
2603                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2604
2605         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
2606                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2607         else
2608                 retval = -1;
2609
2610         return retval;
2611 }
2612
2613 /*
2614  *------------------------------------------------------------------------------
2615  * PMD Framework
2616  *------------------------------------------------------------------------------
2617  */
2618
2619 /** Enqueue burst */
2620 static uint16_t
2621 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
2622                 uint16_t nb_ops)
2623 {
2624         void *sess;
2625         struct openssl_qp *qp = queue_pair;
2626         int i, retval;
2627
2628         for (i = 0; i < nb_ops; i++) {
2629                 sess = get_session(qp, ops[i]);
2630                 if (unlikely(sess == NULL))
2631                         goto enqueue_err;
2632
2633                 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
2634                         retval = process_op(qp, ops[i],
2635                                         (struct openssl_session *) sess);
2636                 else
2637                         retval = process_asym_op(qp, ops[i],
2638                                         (struct openssl_asym_session *) sess);
2639                 if (unlikely(retval < 0))
2640                         goto enqueue_err;
2641         }
2642
2643         qp->stats.enqueued_count += i;
2644         return i;
2645
2646 enqueue_err:
2647         qp->stats.enqueue_err_count++;
2648         return i;
2649 }
2650
2651 /** Dequeue burst */
2652 static uint16_t
2653 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
2654                 uint16_t nb_ops)
2655 {
2656         struct openssl_qp *qp = queue_pair;
2657
2658         unsigned int nb_dequeued = 0;
2659
2660         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
2661                         (void **)ops, nb_ops, NULL);
2662         qp->stats.dequeued_count += nb_dequeued;
2663
2664         return nb_dequeued;
2665 }
2666
2667 /** Create OPENSSL crypto device */
2668 static int
2669 cryptodev_openssl_create(const char *name,
2670                         struct rte_vdev_device *vdev,
2671                         struct rte_cryptodev_pmd_init_params *init_params)
2672 {
2673         struct rte_cryptodev *dev;
2674         struct openssl_private *internals;
2675
2676         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2677         if (dev == NULL) {
2678                 OPENSSL_LOG(ERR, "failed to create cryptodev vdev");
2679                 goto init_error;
2680         }
2681
2682         dev->driver_id = cryptodev_driver_id;
2683         dev->dev_ops = rte_openssl_pmd_ops;
2684
2685         /* register rx/tx burst functions for data path */
2686         dev->dequeue_burst = openssl_pmd_dequeue_burst;
2687         dev->enqueue_burst = openssl_pmd_enqueue_burst;
2688
2689         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2690                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2691                         RTE_CRYPTODEV_FF_CPU_AESNI |
2692                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2693                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2694                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2695                         RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
2696                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP |
2697                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT |
2698                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
2699
2700         internals = dev->data->dev_private;
2701
2702         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
2703
2704         rte_cryptodev_pmd_probing_finish(dev);
2705
2706 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2707         /* Load legacy provider
2708          * Some algorithms are no longer available in earlier version of openssl,
2709          * unless the legacy provider explicitly loaded. e.g. DES
2710          */
2711         ossl_legacy_provider_load();
2712 # endif
2713         return 0;
2714
2715 init_error:
2716         OPENSSL_LOG(ERR, "driver %s: create failed",
2717                         init_params->name);
2718
2719         cryptodev_openssl_remove(vdev);
2720         return -EFAULT;
2721 }
2722
2723 /** Initialise OPENSSL crypto device */
2724 static int
2725 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
2726 {
2727         struct rte_cryptodev_pmd_init_params init_params = {
2728                 "",
2729                 sizeof(struct openssl_private),
2730                 rte_socket_id(),
2731                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2732         };
2733         const char *name;
2734         const char *input_args;
2735
2736         name = rte_vdev_device_name(vdev);
2737         if (name == NULL)
2738                 return -EINVAL;
2739         input_args = rte_vdev_device_args(vdev);
2740
2741         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2742
2743         return cryptodev_openssl_create(name, vdev, &init_params);
2744 }
2745
2746 /** Uninitialise OPENSSL crypto device */
2747 static int
2748 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
2749 {
2750         struct rte_cryptodev *cryptodev;
2751         const char *name;
2752
2753         name = rte_vdev_device_name(vdev);
2754         if (name == NULL)
2755                 return -EINVAL;
2756
2757         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2758         if (cryptodev == NULL)
2759                 return -ENODEV;
2760
2761 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2762         ossl_legacy_provider_unload();
2763 # endif
2764         return rte_cryptodev_pmd_destroy(cryptodev);
2765 }
2766
2767 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
2768         .probe = cryptodev_openssl_probe,
2769         .remove = cryptodev_openssl_remove
2770 };
2771
2772 static struct cryptodev_driver openssl_crypto_drv;
2773
2774 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
2775         cryptodev_openssl_pmd_drv);
2776 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
2777         "max_nb_queue_pairs=<int> "
2778         "socket_id=<int>");
2779 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
2780                 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);
2781 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO);