drivers/crypto/openssl/rte_openssl_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_hexdump.h>
   7 #include <rte_cryptodev.h>
   8 #include <rte_cryptodev_pmd.h>
   9 #include <rte_bus_vdev.h>
  10 #include <rte_malloc.h>
  11 #include <rte_cpuflags.h>
  12
  13 #include <openssl/hmac.h>
  14 #include <openssl/evp.h>
  15
  16 #include "openssl_pmd_private.h"
  17 #include "compat.h"
  18
  19 #define DES_BLOCK_SIZE 8
  20
  21 int openssl_logtype_driver;
  22 static uint8_t cryptodev_driver_id;
  23
  24 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  25 static HMAC_CTX *HMAC_CTX_new(void)
  26 {
  27         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  28
  29         if (ctx != NULL)
  30                 HMAC_CTX_init(ctx);
  31         return ctx;
  32 }
  33
  34 static void HMAC_CTX_free(HMAC_CTX *ctx)
  35 {
  36         if (ctx != NULL) {
  37                 HMAC_CTX_cleanup(ctx);
  38                 OPENSSL_free(ctx);
  39         }
  40 }
  41 #endif
  42
  43 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  44
  45 /*----------------------------------------------------------------------------*/
  46
  47 /**
  48  * Increment counter by 1
  49  * Counter is 64 bit array, big-endian
  50  */
  51 static void
  52 ctr_inc(uint8_t *ctr)
  53 {
  54         uint64_t *ctr64 = (uint64_t *)ctr;
  55
  56         *ctr64 = __builtin_bswap64(*ctr64);
  57         (*ctr64)++;
  58         *ctr64 = __builtin_bswap64(*ctr64);
  59 }
  60
  61 /*
  62  *------------------------------------------------------------------------------
  63  * Session Prepare
  64  *------------------------------------------------------------------------------
  65  */
  66
  67 /** Get xform chain order */
  68 static enum openssl_chain_order
  69 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
  70 {
  71         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
  72
  73         if (xform != NULL) {
  74                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  75                         if (xform->next == NULL)
  76                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
  77                         else if (xform->next->type ==
  78                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
  79                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
  80                 }
  81                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  82                         if (xform->next == NULL)
  83                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
  84                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  85                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
  86                 }
  87                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
  88                         res = OPENSSL_CHAIN_COMBINED;
  89         }
  90
  91         return res;
  92 }
  93
  94 /** Get session cipher key from input cipher key */
  95 static void
  96 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key)
  97 {
  98         memcpy(session_key, input_key, keylen);
  99 }
 100
 101 /** Get key ede 24 bytes standard from input key */
 102 static int
 103 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede)
 104 {
 105         int res = 0;
 106
 107         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 108         switch (keylen) {
 109         case 24:
 110                 memcpy(key_ede, key, 24);
 111                 break;
 112         case 16:
 113                 /* K3 = K1 */
 114                 memcpy(key_ede, key, 16);
 115                 memcpy(key_ede + 16, key, 8);
 116                 break;
 117         case 8:
 118                 /* K1 = K2 = K3 (DES compatibility) */
 119                 memcpy(key_ede, key, 8);
 120                 memcpy(key_ede + 8, key, 8);
 121                 memcpy(key_ede + 16, key, 8);
 122                 break;
 123         default:
 124                 OPENSSL_LOG(ERR, "Unsupported key size");
 125                 res = -EINVAL;
 126         }
 127
 128         return res;
 129 }
 130
 131 /** Get adequate openssl function for input cipher algorithm */
 132 static uint8_t
 133 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 134                 const EVP_CIPHER **algo)
 135 {
 136         int res = 0;
 137
 138         if (algo != NULL) {
 139                 switch (sess_algo) {
 140                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 141                         switch (keylen) {
 142                         case 8:
 143                                 *algo = EVP_des_cbc();
 144                                 break;
 145                         case 16:
 146                                 *algo = EVP_des_ede_cbc();
 147                                 break;
 148                         case 24:
 149                                 *algo = EVP_des_ede3_cbc();
 150                                 break;
 151                         default:
 152                                 res = -EINVAL;
 153                         }
 154                         break;
 155                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 156                         break;
 157                 case RTE_CRYPTO_CIPHER_AES_CBC:
 158                         switch (keylen) {
 159                         case 16:
 160                                 *algo = EVP_aes_128_cbc();
 161                                 break;
 162                         case 24:
 163                                 *algo = EVP_aes_192_cbc();
 164                                 break;
 165                         case 32:
 166                                 *algo = EVP_aes_256_cbc();
 167                                 break;
 168                         default:
 169                                 res = -EINVAL;
 170                         }
 171                         break;
 172                 case RTE_CRYPTO_CIPHER_AES_CTR:
 173                         switch (keylen) {
 174                         case 16:
 175                                 *algo = EVP_aes_128_ctr();
 176                                 break;
 177                         case 24:
 178                                 *algo = EVP_aes_192_ctr();
 179                                 break;
 180                         case 32:
 181                                 *algo = EVP_aes_256_ctr();
 182                                 break;
 183                         default:
 184                                 res = -EINVAL;
 185                         }
 186                         break;
 187                 default:
 188                         res = -EINVAL;
 189                         break;
 190                 }
 191         } else {
 192                 res = -EINVAL;
 193         }
 194
 195         return res;
 196 }
 197
 198 /** Get adequate openssl function for input auth algorithm */
 199 static uint8_t
 200 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 201                 const EVP_MD **algo)
 202 {
 203         int res = 0;
 204
 205         if (algo != NULL) {
 206                 switch (sessalgo) {
 207                 case RTE_CRYPTO_AUTH_MD5:
 208                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 209                         *algo = EVP_md5();
 210                         break;
 211                 case RTE_CRYPTO_AUTH_SHA1:
 212                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 213                         *algo = EVP_sha1();
 214                         break;
 215                 case RTE_CRYPTO_AUTH_SHA224:
 216                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 217                         *algo = EVP_sha224();
 218                         break;
 219                 case RTE_CRYPTO_AUTH_SHA256:
 220                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 221                         *algo = EVP_sha256();
 222                         break;
 223                 case RTE_CRYPTO_AUTH_SHA384:
 224                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 225                         *algo = EVP_sha384();
 226                         break;
 227                 case RTE_CRYPTO_AUTH_SHA512:
 228                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 229                         *algo = EVP_sha512();
 230                         break;
 231                 default:
 232                         res = -EINVAL;
 233                         break;
 234                 }
 235         } else {
 236                 res = -EINVAL;
 237         }
 238
 239         return res;
 240 }
 241
 242 /** Get adequate openssl function for input cipher algorithm */
 243 static uint8_t
 244 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 245                 const EVP_CIPHER **algo)
 246 {
 247         int res = 0;
 248
 249         if (algo != NULL) {
 250                 switch (sess_algo) {
 251                 case RTE_CRYPTO_AEAD_AES_GCM:
 252                         switch (keylen) {
 253                         case 16:
 254                                 *algo = EVP_aes_128_gcm();
 255                                 break;
 256                         case 24:
 257                                 *algo = EVP_aes_192_gcm();
 258                                 break;
 259                         case 32:
 260                                 *algo = EVP_aes_256_gcm();
 261                                 break;
 262                         default:
 263                                 res = -EINVAL;
 264                         }
 265                         break;
 266                 case RTE_CRYPTO_AEAD_AES_CCM:
 267                         switch (keylen) {
 268                         case 16:
 269                                 *algo = EVP_aes_128_ccm();
 270                                 break;
 271                         case 24:
 272                                 *algo = EVP_aes_192_ccm();
 273                                 break;
 274                         case 32:
 275                                 *algo = EVP_aes_256_ccm();
 276                                 break;
 277                         default:
 278                                 res = -EINVAL;
 279                         }
 280                         break;
 281                 default:
 282                         res = -EINVAL;
 283                         break;
 284                 }
 285         } else {
 286                 res = -EINVAL;
 287         }
 288
 289         return res;
 290 }
 291
 292 /* Set session AEAD encryption parameters */
 293 static int
 294 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
 295                 enum rte_crypto_aead_algorithm algo,
 296                 uint8_t tag_len, const uint8_t *key)
 297 {
 298         int iv_type = 0;
 299         unsigned int do_ccm;
 300
 301         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 302         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 303
 304         /* Select AEAD algo */
 305         switch (algo) {
 306         case RTE_CRYPTO_AEAD_AES_GCM:
 307                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 308                 if (tag_len != 16)
 309                         return -EINVAL;
 310                 do_ccm = 0;
 311                 break;
 312         case RTE_CRYPTO_AEAD_AES_CCM:
 313                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 314                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 315                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 316                         return -EINVAL;
 317                 do_ccm = 1;
 318                 break;
 319         default:
 320                 return -ENOTSUP;
 321         }
 322
 323         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 324         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 325
 326         if (get_aead_algo(algo, sess->cipher.key.length,
 327                         &sess->cipher.evp_algo) != 0)
 328                 return -EINVAL;
 329
 330         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 331
 332         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 333
 334         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 335                         NULL, NULL, NULL) <= 0)
 336                 return -EINVAL;
 337
 338         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
 339                         NULL) <= 0)
 340                 return -EINVAL;
 341
 342         if (do_ccm)
 343                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 344                                 tag_len, NULL);
 345
 346         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 347                 return -EINVAL;
 348
 349         return 0;
 350 }
 351
 352 /* Set session AEAD decryption parameters */
 353 static int
 354 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
 355                 enum rte_crypto_aead_algorithm algo,
 356                 uint8_t tag_len, const uint8_t *key)
 357 {
 358         int iv_type = 0;
 359         unsigned int do_ccm = 0;
 360
 361         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 362         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 363
 364         /* Select AEAD algo */
 365         switch (algo) {
 366         case RTE_CRYPTO_AEAD_AES_GCM:
 367                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 368                 if (tag_len != 16)
 369                         return -EINVAL;
 370                 break;
 371         case RTE_CRYPTO_AEAD_AES_CCM:
 372                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 373                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 374                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 375                         return -EINVAL;
 376                 do_ccm = 1;
 377                 break;
 378         default:
 379                 return -ENOTSUP;
 380         }
 381
 382         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 383         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 384
 385         if (get_aead_algo(algo, sess->cipher.key.length,
 386                         &sess->cipher.evp_algo) != 0)
 387                 return -EINVAL;
 388
 389         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 390
 391         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 392
 393         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 394                         NULL, NULL, NULL) <= 0)
 395                 return -EINVAL;
 396
 397         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
 398                         sess->iv.length, NULL) <= 0)
 399                 return -EINVAL;
 400
 401         if (do_ccm)
 402                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 403                                 tag_len, NULL);
 404
 405         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 406                 return -EINVAL;
 407
 408         return 0;
 409 }
 410
 411 /** Set session cipher parameters */
 412 static int
 413 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 414                 const struct rte_crypto_sym_xform *xform)
 415 {
 416         /* Select cipher direction */
 417         sess->cipher.direction = xform->cipher.op;
 418         /* Select cipher key */
 419         sess->cipher.key.length = xform->cipher.key.length;
 420
 421         /* Set IV parameters */
 422         sess->iv.offset = xform->cipher.iv.offset;
 423         sess->iv.length = xform->cipher.iv.length;
 424
 425         /* Select cipher algo */
 426         switch (xform->cipher.algo) {
 427         case RTE_CRYPTO_CIPHER_3DES_CBC:
 428         case RTE_CRYPTO_CIPHER_AES_CBC:
 429         case RTE_CRYPTO_CIPHER_AES_CTR:
 430                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 431                 sess->cipher.algo = xform->cipher.algo;
 432                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 433
 434                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 435                                 &sess->cipher.evp_algo) != 0)
 436                         return -EINVAL;
 437
 438                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 439                         sess->cipher.key.data);
 440                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 441                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 442                                         sess->cipher.evp_algo,
 443                                         NULL, xform->cipher.key.data,
 444                                         NULL) != 1) {
 445                                 return -EINVAL;
 446                         }
 447                 } else if (sess->cipher.direction ==
 448                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 449                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 450                                         sess->cipher.evp_algo,
 451                                         NULL, xform->cipher.key.data,
 452                                         NULL) != 1) {
 453                                 return -EINVAL;
 454                         }
 455                 }
 456
 457                 break;
 458
 459         case RTE_CRYPTO_CIPHER_3DES_CTR:
 460                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 461                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 462
 463                 if (get_cipher_key_ede(xform->cipher.key.data,
 464                                 sess->cipher.key.length,
 465                                 sess->cipher.key.data) != 0)
 466                         return -EINVAL;
 467                 break;
 468
 469         case RTE_CRYPTO_CIPHER_DES_CBC:
 470                 sess->cipher.algo = xform->cipher.algo;
 471                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 472                 sess->cipher.evp_algo = EVP_des_cbc();
 473
 474                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 475                         sess->cipher.key.data);
 476                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 477                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 478                                         sess->cipher.evp_algo,
 479                                         NULL, xform->cipher.key.data,
 480                                         NULL) != 1) {
 481                                 return -EINVAL;
 482                         }
 483                 } else if (sess->cipher.direction ==
 484                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 485                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 486                                         sess->cipher.evp_algo,
 487                                         NULL, xform->cipher.key.data,
 488                                         NULL) != 1) {
 489                                 return -EINVAL;
 490                         }
 491                 }
 492
 493                 break;
 494
 495         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 496                 sess->cipher.algo = xform->cipher.algo;
 497                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 498                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 499                 sess->cipher.evp_algo = EVP_des_cbc();
 500
 501                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 502                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 503                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 504                                 NULL, xform->cipher.key.data, 0) != 1)
 505                         return -EINVAL;
 506
 507                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 508                         sess->cipher.key.data);
 509                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 510                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 511                                         sess->cipher.evp_algo,
 512                                         NULL, xform->cipher.key.data,
 513                                         NULL) != 1) {
 514                                 return -EINVAL;
 515                         }
 516                 } else if (sess->cipher.direction ==
 517                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 518                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 519                                         sess->cipher.evp_algo,
 520                                         NULL, xform->cipher.key.data,
 521                                         NULL) != 1) {
 522                                 return -EINVAL;
 523                         }
 524                 }
 525
 526                 break;
 527         default:
 528                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 529                 return -ENOTSUP;
 530         }
 531
 532         return 0;
 533 }
 534
 535 /* Set session auth parameters */
 536 static int
 537 openssl_set_session_auth_parameters(struct openssl_session *sess,
 538                 const struct rte_crypto_sym_xform *xform)
 539 {
 540         /* Select auth generate/verify */
 541         sess->auth.operation = xform->auth.op;
 542         sess->auth.algo = xform->auth.algo;
 543
 544         sess->auth.digest_length = xform->auth.digest_length;
 545
 546         /* Select auth algo */
 547         switch (xform->auth.algo) {
 548         case RTE_CRYPTO_AUTH_AES_GMAC:
 549                 /*
 550                  * OpenSSL requires GMAC to be a GCM operation
 551                  * with no cipher data length
 552                  */
 553                 sess->cipher.key.length = xform->auth.key.length;
 554
 555                 /* Set IV parameters */
 556                 sess->iv.offset = xform->auth.iv.offset;
 557                 sess->iv.length = xform->auth.iv.length;
 558
 559                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 560                         return openssl_set_sess_aead_enc_param(sess,
 561                                                 RTE_CRYPTO_AEAD_AES_GCM,
 562                                                 xform->auth.digest_length,
 563                                                 xform->auth.key.data);
 564                 else
 565                         return openssl_set_sess_aead_dec_param(sess,
 566                                                 RTE_CRYPTO_AEAD_AES_GCM,
 567                                                 xform->auth.digest_length,
 568                                                 xform->auth.key.data);
 569                 break;
 570
 571         case RTE_CRYPTO_AUTH_MD5:
 572         case RTE_CRYPTO_AUTH_SHA1:
 573         case RTE_CRYPTO_AUTH_SHA224:
 574         case RTE_CRYPTO_AUTH_SHA256:
 575         case RTE_CRYPTO_AUTH_SHA384:
 576         case RTE_CRYPTO_AUTH_SHA512:
 577                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 578                 if (get_auth_algo(xform->auth.algo,
 579                                 &sess->auth.auth.evp_algo) != 0)
 580                         return -EINVAL;
 581                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 582                 break;
 583
 584         case RTE_CRYPTO_AUTH_MD5_HMAC:
 585         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 586         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 587         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 588         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 589         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 590                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 591                 sess->auth.hmac.ctx = HMAC_CTX_new();
 592                 if (get_auth_algo(xform->auth.algo,
 593                                 &sess->auth.hmac.evp_algo) != 0)
 594                         return -EINVAL;
 595
 596                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 597                                 xform->auth.key.data,
 598                                 xform->auth.key.length,
 599                                 sess->auth.hmac.evp_algo, NULL) != 1)
 600                         return -EINVAL;
 601                 break;
 602
 603         default:
 604                 return -ENOTSUP;
 605         }
 606
 607         return 0;
 608 }
 609
 610 /* Set session AEAD parameters */
 611 static int
 612 openssl_set_session_aead_parameters(struct openssl_session *sess,
 613                 const struct rte_crypto_sym_xform *xform)
 614 {
 615         /* Select cipher key */
 616         sess->cipher.key.length = xform->aead.key.length;
 617
 618         /* Set IV parameters */
 619         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 620                 /*
 621                  * For AES-CCM, the actual IV is placed
 622                  * one byte after the start of the IV field,
 623                  * according to the API.
 624                  */
 625                 sess->iv.offset = xform->aead.iv.offset + 1;
 626         else
 627                 sess->iv.offset = xform->aead.iv.offset;
 628
 629         sess->iv.length = xform->aead.iv.length;
 630
 631         sess->auth.aad_length = xform->aead.aad_length;
 632         sess->auth.digest_length = xform->aead.digest_length;
 633
 634         sess->aead_algo = xform->aead.algo;
 635         /* Select cipher direction */
 636         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 637                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
 638                                 xform->aead.digest_length, xform->aead.key.data);
 639         else
 640                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
 641                                 xform->aead.digest_length, xform->aead.key.data);
 642 }
 643
 644 /** Parse crypto xform chain and set private session parameters */
 645 int
 646 openssl_set_session_parameters(struct openssl_session *sess,
 647                 const struct rte_crypto_sym_xform *xform)
 648 {
 649         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 650         const struct rte_crypto_sym_xform *auth_xform = NULL;
 651         const struct rte_crypto_sym_xform *aead_xform = NULL;
 652         int ret;
 653
 654         sess->chain_order = openssl_get_chain_order(xform);
 655         switch (sess->chain_order) {
 656         case OPENSSL_CHAIN_ONLY_CIPHER:
 657                 cipher_xform = xform;
 658                 break;
 659         case OPENSSL_CHAIN_ONLY_AUTH:
 660                 auth_xform = xform;
 661                 break;
 662         case OPENSSL_CHAIN_CIPHER_AUTH:
 663                 cipher_xform = xform;
 664                 auth_xform = xform->next;
 665                 break;
 666         case OPENSSL_CHAIN_AUTH_CIPHER:
 667                 auth_xform = xform;
 668                 cipher_xform = xform->next;
 669                 break;
 670         case OPENSSL_CHAIN_COMBINED:
 671                 aead_xform = xform;
 672                 break;
 673         default:
 674                 return -EINVAL;
 675         }
 676
 677         /* Default IV length = 0 */
 678         sess->iv.length = 0;
 679
 680         /* cipher_xform must be check before auth_xform */
 681         if (cipher_xform) {
 682                 ret = openssl_set_session_cipher_parameters(
 683                                 sess, cipher_xform);
 684                 if (ret != 0) {
 685                         OPENSSL_LOG(ERR,
 686                                 "Invalid/unsupported cipher parameters");
 687                         return ret;
 688                 }
 689         }
 690
 691         if (auth_xform) {
 692                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 693                 if (ret != 0) {
 694                         OPENSSL_LOG(ERR,
 695                                 "Invalid/unsupported auth parameters");
 696                         return ret;
 697                 }
 698         }
 699
 700         if (aead_xform) {
 701                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 702                 if (ret != 0) {
 703                         OPENSSL_LOG(ERR,
 704                                 "Invalid/unsupported AEAD parameters");
 705                         return ret;
 706                 }
 707         }
 708
 709         return 0;
 710 }
 711
 712 /** Reset private session parameters */
 713 void
 714 openssl_reset_session(struct openssl_session *sess)
 715 {
 716         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 717
 718         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 719                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 720
 721         switch (sess->auth.mode) {
 722         case OPENSSL_AUTH_AS_AUTH:
 723                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 724                 break;
 725         case OPENSSL_AUTH_AS_HMAC:
 726                 EVP_PKEY_free(sess->auth.hmac.pkey);
 727                 HMAC_CTX_free(sess->auth.hmac.ctx);
 728                 break;
 729         default:
 730                 break;
 731         }
 732 }
 733
 734 /** Provide session for operation */
 735 static void *
 736 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 737 {
 738         struct openssl_session *sess = NULL;
 739         struct openssl_asym_session *asym_sess = NULL;
 740
 741         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 742                 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 743                         /* get existing session */
 744                         if (likely(op->sym->session != NULL))
 745                                 sess = (struct openssl_session *)
 746                                                 get_sym_session_private_data(
 747                                                 op->sym->session,
 748                                                 cryptodev_driver_id);
 749                 } else {
 750                         if (likely(op->asym->session != NULL))
 751                                 asym_sess = (struct openssl_asym_session *)
 752                                                 get_asym_session_private_data(
 753                                                 op->asym->session,
 754                                                 cryptodev_driver_id);
 755                         if (asym_sess == NULL)
 756                                 op->status =
 757                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 758                         return asym_sess;
 759                 }
 760         } else {
 761                 /* sessionless asymmetric not supported */
 762                 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
 763                         return NULL;
 764
 765                 /* provide internal session */
 766                 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
 767                 void *_sess_private_data = NULL;
 768
 769                 if (_sess == NULL)
 770                         return NULL;
 771
 772                 if (rte_mempool_get(qp->sess_mp_priv,
 773                                 (void **)&_sess_private_data))
 774                         return NULL;
 775
 776                 sess = (struct openssl_session *)_sess_private_data;
 777
 778                 if (unlikely(openssl_set_session_parameters(sess,
 779                                 op->sym->xform) != 0)) {
 780                         rte_mempool_put(qp->sess_mp, _sess);
 781                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 782                         sess = NULL;
 783                 }
 784                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 785                 set_sym_session_private_data(op->sym->session,
 786                                 cryptodev_driver_id, _sess_private_data);
 787         }
 788
 789         if (sess == NULL)
 790                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 791
 792         return sess;
 793 }
 794
 795 /*
 796  *------------------------------------------------------------------------------
 797  * Process Operations
 798  *------------------------------------------------------------------------------
 799  */
 800 static inline int
 801 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 802                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
 803 {
 804         struct rte_mbuf *m;
 805         int dstlen;
 806         int l, n = srclen;
 807         uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
 808
 809         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 810                         m = m->next)
 811                 offset -= rte_pktmbuf_data_len(m);
 812
 813         if (m == 0)
 814                 return -1;
 815
 816         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 817         if (inplace)
 818                 *dst = src;
 819
 820         l = rte_pktmbuf_data_len(m) - offset;
 821         if (srclen <= l) {
 822                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 823                         return -1;
 824                 *dst += l;
 825                 return 0;
 826         }
 827
 828         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 829                 return -1;
 830
 831         *dst += dstlen;
 832         n -= l;
 833
 834         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 835                 uint8_t diff = l - dstlen, rem;
 836
 837                 src = rte_pktmbuf_mtod(m, uint8_t *);
 838                 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
 839                 if (diff && inplace) {
 840                         rem = RTE_MIN(l,
 841                                 (EVP_CIPHER_CTX_block_size(ctx) - diff));
 842                         if (EVP_EncryptUpdate(ctx, temp,
 843                                                 &dstlen, src, rem) <= 0)
 844                                 return -1;
 845                         n -= rem;
 846                         rte_memcpy(*dst, temp, diff);
 847                         rte_memcpy(src, temp + diff, rem);
 848                         src += rem;
 849                         l -= rem;
 850                 }
 851                 if (inplace)
 852                         *dst = src;
 853                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 854                         return -1;
 855                 *dst += dstlen;
 856                 n -= l;
 857         }
 858
 859         return 0;
 860 }
 861
 862 static inline int
 863 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 864                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
 865 {
 866         struct rte_mbuf *m;
 867         int dstlen;
 868         int l, n = srclen;
 869         uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
 870
 871         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 872                         m = m->next)
 873                 offset -= rte_pktmbuf_data_len(m);
 874
 875         if (m == 0)
 876                 return -1;
 877
 878         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 879         if (inplace)
 880                 *dst = src;
 881
 882         l = rte_pktmbuf_data_len(m) - offset;
 883         if (srclen <= l) {
 884                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 885                         return -1;
 886                 *dst += l;
 887                 return 0;
 888         }
 889
 890         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 891                 return -1;
 892
 893         *dst += dstlen;
 894         n -= l;
 895
 896         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 897                 uint8_t diff = l - dstlen, rem;
 898
 899                 src = rte_pktmbuf_mtod(m, uint8_t *);
 900                 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
 901                 if (diff && inplace) {
 902                         rem = RTE_MIN(l,
 903                                 (EVP_CIPHER_CTX_block_size(ctx) - diff));
 904                         if (EVP_DecryptUpdate(ctx, temp,
 905                                                 &dstlen, src, rem) <= 0)
 906                                 return -1;
 907                         n -= rem;
 908                         rte_memcpy(*dst, temp, diff);
 909                         rte_memcpy(src, temp + diff, rem);
 910                         src += rem;
 911                         l -= rem;
 912                 }
 913                 if (inplace)
 914                         *dst = src;
 915                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 916                         return -1;
 917                 *dst += dstlen;
 918                 n -= l;
 919         }
 920
 921         return 0;
 922 }
 923
 924 /** Process standard openssl cipher encryption */
 925 static int
 926 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 927                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
 928                 uint8_t inplace)
 929 {
 930         int totlen;
 931
 932         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 933                 goto process_cipher_encrypt_err;
 934
 935         EVP_CIPHER_CTX_set_padding(ctx, 0);
 936
 937         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 938                         srclen, ctx, inplace))
 939                 goto process_cipher_encrypt_err;
 940
 941         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
 942                 goto process_cipher_encrypt_err;
 943
 944         return 0;
 945
 946 process_cipher_encrypt_err:
 947         OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed");
 948         return -EINVAL;
 949 }
 950
 951 /** Process standard openssl cipher encryption */
 952 static int
 953 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 954                 uint8_t *iv, int srclen,
 955                 EVP_CIPHER_CTX *ctx)
 956 {
 957         uint8_t i;
 958         uint8_t encrypted_iv[DES_BLOCK_SIZE];
 959         int encrypted_ivlen;
 960
 961         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
 962                         iv, DES_BLOCK_SIZE) <= 0)
 963                 goto process_cipher_encrypt_err;
 964
 965         for (i = 0; i < srclen; i++)
 966                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
 967
 968         return 0;
 969
 970 process_cipher_encrypt_err:
 971         OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed");
 972         return -EINVAL;
 973 }
 974 /** Process standard openssl cipher decryption */
 975 static int
 976 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 977                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
 978                 uint8_t inplace)
 979 {
 980         int totlen;
 981
 982         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 983                 goto process_cipher_decrypt_err;
 984
 985         EVP_CIPHER_CTX_set_padding(ctx, 0);
 986
 987         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 988                         srclen, ctx, inplace))
 989                 goto process_cipher_decrypt_err;
 990
 991         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
 992                 goto process_cipher_decrypt_err;
 993         return 0;
 994
 995 process_cipher_decrypt_err:
 996         OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed");
 997         return -EINVAL;
 998 }
 999
1000 /** Process cipher des 3 ctr encryption, decryption algorithm */
1001 static int
1002 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
1003                 int offset, uint8_t *iv, uint8_t *key, int srclen,
1004                 EVP_CIPHER_CTX *ctx)
1005 {
1006         uint8_t ebuf[8], ctr[8];
1007         int unused, n;
1008         struct rte_mbuf *m;
1009         uint8_t *src;
1010         int l;
1011
1012         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1013                         m = m->next)
1014                 offset -= rte_pktmbuf_data_len(m);
1015
1016         if (m == 0)
1017                 goto process_cipher_des3ctr_err;
1018
1019         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1020         l = rte_pktmbuf_data_len(m) - offset;
1021
1022         /* We use 3DES encryption also for decryption.
1023          * IV is not important for 3DES ecb
1024          */
1025         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
1026                 goto process_cipher_des3ctr_err;
1027
1028         memcpy(ctr, iv, 8);
1029
1030         for (n = 0; n < srclen; n++) {
1031                 if (n % 8 == 0) {
1032                         if (EVP_EncryptUpdate(ctx,
1033                                         (unsigned char *)&ebuf, &unused,
1034                                         (const unsigned char *)&ctr, 8) <= 0)
1035                                 goto process_cipher_des3ctr_err;
1036                         ctr_inc(ctr);
1037                 }
1038                 dst[n] = *(src++) ^ ebuf[n % 8];
1039
1040                 l--;
1041                 if (!l) {
1042                         m = m->next;
1043                         if (m) {
1044                                 src = rte_pktmbuf_mtod(m, uint8_t *);
1045                                 l = rte_pktmbuf_data_len(m);
1046                         }
1047                 }
1048         }
1049
1050         return 0;
1051
1052 process_cipher_des3ctr_err:
1053         OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed");
1054         return -EINVAL;
1055 }
1056
1057 /** Process AES-GCM encrypt algorithm */
1058 static int
1059 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1060                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1061                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1062 {
1063         int len = 0, unused = 0;
1064         uint8_t empty[] = {};
1065
1066         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1067                 goto process_auth_encryption_gcm_err;
1068
1069         if (aadlen > 0)
1070                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1071                         goto process_auth_encryption_gcm_err;
1072
1073         if (srclen > 0)
1074                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1075                                 srclen, ctx, 0))
1076                         goto process_auth_encryption_gcm_err;
1077
1078         /* Workaround open ssl bug in version less then 1.0.1f */
1079         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1080                 goto process_auth_encryption_gcm_err;
1081
1082         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1083                 goto process_auth_encryption_gcm_err;
1084
1085         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1086                 goto process_auth_encryption_gcm_err;
1087
1088         return 0;
1089
1090 process_auth_encryption_gcm_err:
1091         OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed");
1092         return -EINVAL;
1093 }
1094
1095 /** Process AES-CCM encrypt algorithm */
1096 static int
1097 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1098                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1099                 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1100 {
1101         int len = 0;
1102
1103         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1104                 goto process_auth_encryption_ccm_err;
1105
1106         if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1107                 goto process_auth_encryption_ccm_err;
1108
1109         if (aadlen > 0)
1110                 /*
1111                  * For AES-CCM, the actual AAD is placed
1112                  * 18 bytes after the start of the AAD field,
1113                  * according to the API.
1114                  */
1115                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1116                         goto process_auth_encryption_ccm_err;
1117
1118         if (srclen > 0)
1119                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1120                                 srclen, ctx, 0))
1121                         goto process_auth_encryption_ccm_err;
1122
1123         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1124                 goto process_auth_encryption_ccm_err;
1125
1126         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1127                 goto process_auth_encryption_ccm_err;
1128
1129         return 0;
1130
1131 process_auth_encryption_ccm_err:
1132         OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed");
1133         return -EINVAL;
1134 }
1135
1136 /** Process AES-GCM decrypt algorithm */
1137 static int
1138 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1139                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1140                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1141 {
1142         int len = 0, unused = 0;
1143         uint8_t empty[] = {};
1144
1145         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1146                 goto process_auth_decryption_gcm_err;
1147
1148         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1149                 goto process_auth_decryption_gcm_err;
1150
1151         if (aadlen > 0)
1152                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1153                         goto process_auth_decryption_gcm_err;
1154
1155         if (srclen > 0)
1156                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1157                                 srclen, ctx, 0))
1158                         goto process_auth_decryption_gcm_err;
1159
1160         /* Workaround open ssl bug in version less then 1.0.1f */
1161         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1162                 goto process_auth_decryption_gcm_err;
1163
1164         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1165                 return -EFAULT;
1166
1167         return 0;
1168
1169 process_auth_decryption_gcm_err:
1170         OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed");
1171         return -EINVAL;
1172 }
1173
1174 /** Process AES-CCM decrypt algorithm */
1175 static int
1176 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1177                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1178                 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1179                 EVP_CIPHER_CTX *ctx)
1180 {
1181         int len = 0;
1182
1183         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1184                 goto process_auth_decryption_ccm_err;
1185
1186         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1187                 goto process_auth_decryption_ccm_err;
1188
1189         if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1190                 goto process_auth_decryption_ccm_err;
1191
1192         if (aadlen > 0)
1193                 /*
1194                  * For AES-CCM, the actual AAD is placed
1195                  * 18 bytes after the start of the AAD field,
1196                  * according to the API.
1197                  */
1198                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1199                         goto process_auth_decryption_ccm_err;
1200
1201         if (srclen > 0)
1202                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1203                                 srclen, ctx, 0))
1204                         return -EFAULT;
1205
1206         return 0;
1207
1208 process_auth_decryption_ccm_err:
1209         OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed");
1210         return -EINVAL;
1211 }
1212
1213 /** Process standard openssl auth algorithms */
1214 static int
1215 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1216                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1217                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1218 {
1219         size_t dstlen;
1220         struct rte_mbuf *m;
1221         int l, n = srclen;
1222         uint8_t *src;
1223
1224         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1225                         m = m->next)
1226                 offset -= rte_pktmbuf_data_len(m);
1227
1228         if (m == 0)
1229                 goto process_auth_err;
1230
1231         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1232                 goto process_auth_err;
1233
1234         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1235
1236         l = rte_pktmbuf_data_len(m) - offset;
1237         if (srclen <= l) {
1238                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1239                         goto process_auth_err;
1240                 goto process_auth_final;
1241         }
1242
1243         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1244                 goto process_auth_err;
1245
1246         n -= l;
1247
1248         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1249                 src = rte_pktmbuf_mtod(m, uint8_t *);
1250                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1251                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1252                         goto process_auth_err;
1253                 n -= l;
1254         }
1255
1256 process_auth_final:
1257         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1258                 goto process_auth_err;
1259         return 0;
1260
1261 process_auth_err:
1262         OPENSSL_LOG(ERR, "Process openssl auth failed");
1263         return -EINVAL;
1264 }
1265
1266 /** Process standard openssl auth algorithms with hmac */
1267 static int
1268 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1269                 int srclen, HMAC_CTX *ctx)
1270 {
1271         unsigned int dstlen;
1272         struct rte_mbuf *m;
1273         int l, n = srclen;
1274         uint8_t *src;
1275
1276         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1277                         m = m->next)
1278                 offset -= rte_pktmbuf_data_len(m);
1279
1280         if (m == 0)
1281                 goto process_auth_err;
1282
1283         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1284
1285         l = rte_pktmbuf_data_len(m) - offset;
1286         if (srclen <= l) {
1287                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1288                         goto process_auth_err;
1289                 goto process_auth_final;
1290         }
1291
1292         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1293                 goto process_auth_err;
1294
1295         n -= l;
1296
1297         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1298                 src = rte_pktmbuf_mtod(m, uint8_t *);
1299                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1300                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1301                         goto process_auth_err;
1302                 n -= l;
1303         }
1304
1305 process_auth_final:
1306         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1307                 goto process_auth_err;
1308
1309         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1310                 goto process_auth_err;
1311
1312         return 0;
1313
1314 process_auth_err:
1315         OPENSSL_LOG(ERR, "Process openssl auth failed");
1316         return -EINVAL;
1317 }
1318
1319 /*----------------------------------------------------------------------------*/
1320
1321 /** Process auth/cipher combined operation */
1322 static void
1323 process_openssl_combined_op
1324                 (struct rte_crypto_op *op, struct openssl_session *sess,
1325                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1326 {
1327         /* cipher */
1328         uint8_t *dst = NULL, *iv, *tag, *aad;
1329         int srclen, aadlen, status = -1;
1330         uint32_t offset;
1331         uint8_t taglen;
1332         EVP_CIPHER_CTX *ctx_copy;
1333
1334         /*
1335          * Segmented destination buffer is not supported for
1336          * encryption/decryption
1337          */
1338         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1339                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1340                 return;
1341         }
1342
1343         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1344                         sess->iv.offset);
1345         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1346                 srclen = 0;
1347                 offset = op->sym->auth.data.offset;
1348                 aadlen = op->sym->auth.data.length;
1349                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1350                                 op->sym->auth.data.offset);
1351                 tag = op->sym->auth.digest.data;
1352                 if (tag == NULL)
1353                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1354                                 offset + aadlen);
1355         } else {
1356                 srclen = op->sym->aead.data.length;
1357                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1358                                 op->sym->aead.data.offset);
1359                 offset = op->sym->aead.data.offset;
1360                 aad = op->sym->aead.aad.data;
1361                 aadlen = sess->auth.aad_length;
1362                 tag = op->sym->aead.digest.data;
1363                 if (tag == NULL)
1364                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1365                                 offset + srclen);
1366         }
1367
1368         taglen = sess->auth.digest_length;
1369         ctx_copy = EVP_CIPHER_CTX_new();
1370         EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx);
1371
1372         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1373                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1374                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1375                         status = process_openssl_auth_encryption_gcm(
1376                                         mbuf_src, offset, srclen,
1377                                         aad, aadlen, iv,
1378                                         dst, tag, ctx_copy);
1379                 else
1380                         status = process_openssl_auth_encryption_ccm(
1381                                         mbuf_src, offset, srclen,
1382                                         aad, aadlen, iv,
1383                                         dst, tag, taglen, ctx_copy);
1384
1385         } else {
1386                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1387                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1388                         status = process_openssl_auth_decryption_gcm(
1389                                         mbuf_src, offset, srclen,
1390                                         aad, aadlen, iv,
1391                                         dst, tag, ctx_copy);
1392                 else
1393                         status = process_openssl_auth_decryption_ccm(
1394                                         mbuf_src, offset, srclen,
1395                                         aad, aadlen, iv,
1396                                         dst, tag, taglen, ctx_copy);
1397         }
1398
1399         EVP_CIPHER_CTX_free(ctx_copy);
1400         if (status != 0) {
1401                 if (status == (-EFAULT) &&
1402                                 sess->auth.operation ==
1403                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1404                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1405                 else
1406                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1407         }
1408 }
1409
1410 /** Process cipher operation */
1411 static void
1412 process_openssl_cipher_op
1413                 (struct rte_crypto_op *op, struct openssl_session *sess,
1414                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1415 {
1416         uint8_t *dst, *iv;
1417         int srclen, status;
1418         uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0;
1419         EVP_CIPHER_CTX *ctx_copy;
1420
1421         /*
1422          * Segmented OOP destination buffer is not supported for encryption/
1423          * decryption. In case of des3ctr, even inplace segmented buffers are
1424          * not supported.
1425          */
1426         if (!rte_pktmbuf_is_contiguous(mbuf_dst) &&
1427                         (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) {
1428                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1429                 return;
1430         }
1431
1432         srclen = op->sym->cipher.data.length;
1433         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1434                         op->sym->cipher.data.offset);
1435
1436         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1437                         sess->iv.offset);
1438         ctx_copy = EVP_CIPHER_CTX_new();
1439         EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx);
1440
1441         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1442                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1443                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1444                                         op->sym->cipher.data.offset, iv,
1445                                         srclen, ctx_copy, inplace);
1446                 else
1447                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1448                                         op->sym->cipher.data.offset, iv,
1449                                         srclen, ctx_copy, inplace);
1450         else
1451                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1452                                 op->sym->cipher.data.offset, iv,
1453                                 sess->cipher.key.data, srclen,
1454                                 ctx_copy);
1455
1456         EVP_CIPHER_CTX_free(ctx_copy);
1457         if (status != 0)
1458                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1459 }
1460
1461 /** Process cipher operation */
1462 static void
1463 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1464                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1465                 struct rte_mbuf *mbuf_dst)
1466 {
1467         uint8_t *src, *dst, *iv;
1468         uint8_t block_size, last_block_len;
1469         int srclen, status = 0;
1470
1471         srclen = op->sym->cipher.data.length;
1472         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1473                         op->sym->cipher.data.offset);
1474         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1475                         op->sym->cipher.data.offset);
1476
1477         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1478                         sess->iv.offset);
1479
1480         block_size = DES_BLOCK_SIZE;
1481
1482         last_block_len = srclen % block_size;
1483         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1484                 /* Encrypt only with ECB mode XOR IV */
1485                 if (srclen < block_size) {
1486                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1487                                         iv, srclen,
1488                                         sess->cipher.bpi_ctx);
1489                 } else {
1490                         srclen -= last_block_len;
1491                         /* Encrypt with the block aligned stream with CBC mode */
1492                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1493                                         op->sym->cipher.data.offset, iv,
1494                                         srclen, sess->cipher.ctx, 0);
1495                         if (last_block_len) {
1496                                 /* Point at last block */
1497                                 dst += srclen;
1498                                 /*
1499                                  * IV is the last encrypted block from
1500                                  * the previous operation
1501                                  */
1502                                 iv = dst - block_size;
1503                                 src += srclen;
1504                                 srclen = last_block_len;
1505                                 /* Encrypt the last frame with ECB mode */
1506                                 status |= process_openssl_cipher_bpi_encrypt(src,
1507                                                 dst, iv,
1508                                                 srclen, sess->cipher.bpi_ctx);
1509                         }
1510                 }
1511         } else {
1512                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1513                 if (srclen < block_size) {
1514                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1515                                         iv,
1516                                         srclen,
1517                                         sess->cipher.bpi_ctx);
1518                 } else {
1519                         if (last_block_len) {
1520                                 /* Point at last block */
1521                                 dst += srclen - last_block_len;
1522                                 src += srclen - last_block_len;
1523                                 /*
1524                                  * IV is the last full block
1525                                  */
1526                                 iv = src - block_size;
1527                                 /*
1528                                  * Decrypt the last frame with ECB mode
1529                                  * (encrypt, as it is the same operation)
1530                                  */
1531                                 status = process_openssl_cipher_bpi_encrypt(src,
1532                                                 dst, iv,
1533                                                 last_block_len, sess->cipher.bpi_ctx);
1534                                 /* Prepare parameters for CBC mode op */
1535                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1536                                                 sess->iv.offset);
1537                                 dst += last_block_len - srclen;
1538                                 srclen -= last_block_len;
1539                         }
1540
1541                         /* Decrypt with CBC mode */
1542                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1543                                         op->sym->cipher.data.offset, iv,
1544                                         srclen, sess->cipher.ctx, 0);
1545                 }
1546         }
1547
1548         if (status != 0)
1549                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1550 }
1551
1552 /** Process auth operation */
1553 static void
1554 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1555                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1556                 struct rte_mbuf *mbuf_dst)
1557 {
1558         uint8_t *dst;
1559         int srclen, status;
1560         EVP_MD_CTX *ctx_a;
1561         HMAC_CTX *ctx_h;
1562
1563         srclen = op->sym->auth.data.length;
1564
1565         dst = qp->temp_digest;
1566
1567         switch (sess->auth.mode) {
1568         case OPENSSL_AUTH_AS_AUTH:
1569                 ctx_a = EVP_MD_CTX_create();
1570                 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx);
1571                 status = process_openssl_auth(mbuf_src, dst,
1572                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1573                                 ctx_a, sess->auth.auth.evp_algo);
1574                 EVP_MD_CTX_destroy(ctx_a);
1575                 break;
1576         case OPENSSL_AUTH_AS_HMAC:
1577                 ctx_h = HMAC_CTX_new();
1578                 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx);
1579                 status = process_openssl_auth_hmac(mbuf_src, dst,
1580                                 op->sym->auth.data.offset, srclen,
1581                                 ctx_h);
1582                 HMAC_CTX_free(ctx_h);
1583                 break;
1584         default:
1585                 status = -1;
1586                 break;
1587         }
1588
1589         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1590                 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data,
1591                                 sess->auth.digest_length) != 0) {
1592                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1593                 }
1594         } else {
1595                 uint8_t *auth_dst;
1596
1597                 auth_dst = op->sym->auth.digest.data;
1598                 if (auth_dst == NULL)
1599                         auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1600                                         op->sym->auth.data.offset +
1601                                         op->sym->auth.data.length);
1602                 memcpy(auth_dst, dst, sess->auth.digest_length);
1603         }
1604
1605         if (status != 0)
1606                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1607 }
1608
1609 /* process dsa sign operation */
1610 static int
1611 process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
1612                 struct openssl_asym_session *sess)
1613 {
1614         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1615         DSA *dsa = sess->u.s.dsa;
1616         DSA_SIG *sign = NULL;
1617
1618         sign = DSA_do_sign(op->message.data,
1619                         op->message.length,
1620                         dsa);
1621
1622         if (sign == NULL) {
1623                 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
1624                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1625         } else {
1626                 const BIGNUM *r = NULL, *s = NULL;
1627                 get_dsa_sign(sign, &r, &s);
1628
1629                 op->r.length = BN_bn2bin(r, op->r.data);
1630                 op->s.length = BN_bn2bin(s, op->s.data);
1631                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1632         }
1633
1634         DSA_SIG_free(sign);
1635
1636         return 0;
1637 }
1638
1639 /* process dsa verify operation */
1640 static int
1641 process_openssl_dsa_verify_op(struct rte_crypto_op *cop,
1642                 struct openssl_asym_session *sess)
1643 {
1644         struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1645         DSA *dsa = sess->u.s.dsa;
1646         int ret;
1647         DSA_SIG *sign = DSA_SIG_new();
1648         BIGNUM *r = NULL, *s = NULL;
1649         BIGNUM *pub_key = NULL;
1650
1651         if (sign == NULL) {
1652                 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
1653                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1654                 return -1;
1655         }
1656
1657         r = BN_bin2bn(op->r.data,
1658                         op->r.length,
1659                         r);
1660         s = BN_bin2bn(op->s.data,
1661                         op->s.length,
1662                         s);
1663         pub_key = BN_bin2bn(op->y.data,
1664                         op->y.length,
1665                         pub_key);
1666         if (!r || !s || !pub_key) {
1667                 BN_free(r);
1668                 BN_free(s);
1669                 BN_free(pub_key);
1670
1671                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1672                 return -1;
1673         }
1674         set_dsa_sign(sign, r, s);
1675         set_dsa_pub_key(dsa, pub_key);
1676
1677         ret = DSA_do_verify(op->message.data,
1678                         op->message.length,
1679                         sign,
1680                         dsa);
1681
1682         if (ret != 1)
1683                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1684         else
1685                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1686
1687         DSA_SIG_free(sign);
1688
1689         return 0;
1690 }
1691
1692 /* process dh operation */
1693 static int
1694 process_openssl_dh_op(struct rte_crypto_op *cop,
1695                 struct openssl_asym_session *sess)
1696 {
1697         struct rte_crypto_dh_op_param *op = &cop->asym->dh;
1698         DH *dh_key = sess->u.dh.dh_key;
1699         BIGNUM *priv_key = NULL;
1700         int ret = 0;
1701
1702         if (sess->u.dh.key_op &
1703                         (1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) {
1704                 /* compute shared secret using peer public key
1705                  * and current private key
1706                  * shared secret = peer_key ^ priv_key mod p
1707                  */
1708                 BIGNUM *peer_key = NULL;
1709
1710                 /* copy private key and peer key and compute shared secret */
1711                 peer_key = BN_bin2bn(op->pub_key.data,
1712                                 op->pub_key.length,
1713                                 peer_key);
1714                 if (peer_key == NULL) {
1715                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1716                         return -1;
1717                 }
1718                 priv_key = BN_bin2bn(op->priv_key.data,
1719                                 op->priv_key.length,
1720                                 priv_key);
1721                 if (priv_key == NULL) {
1722                         BN_free(peer_key);
1723                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1724                         return -1;
1725                 }
1726                 ret = set_dh_priv_key(dh_key, priv_key);
1727                 if (ret) {
1728                         OPENSSL_LOG(ERR, "Failed to set private key\n");
1729                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1730                         BN_free(peer_key);
1731                         BN_free(priv_key);
1732                         return 0;
1733                 }
1734
1735                 ret = DH_compute_key(
1736                                 op->shared_secret.data,
1737                                 peer_key, dh_key);
1738                 if (ret < 0) {
1739                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1740                         BN_free(peer_key);
1741                         /* priv key is already loaded into dh,
1742                          * let's not free that directly here.
1743                          * DH_free() will auto free it later.
1744                          */
1745                         return 0;
1746                 }
1747                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1748                 op->shared_secret.length = ret;
1749                 BN_free(peer_key);
1750                 return 0;
1751         }
1752
1753         /*
1754          * other options are public and private key generations.
1755          *
1756          * if user provides private key,
1757          * then first set DH with user provided private key
1758          */
1759         if ((sess->u.dh.key_op &
1760                         (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) &&
1761                         !(sess->u.dh.key_op &
1762                         (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) {
1763                 /* generate public key using user-provided private key
1764                  * pub_key = g ^ priv_key mod p
1765                  */
1766
1767                 /* load private key into DH */
1768                 priv_key = BN_bin2bn(op->priv_key.data,
1769                                 op->priv_key.length,
1770                                 priv_key);
1771                 if (priv_key == NULL) {
1772                         cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1773                         return -1;
1774                 }
1775                 ret = set_dh_priv_key(dh_key, priv_key);
1776                 if (ret) {
1777                         OPENSSL_LOG(ERR, "Failed to set private key\n");
1778                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1779                         BN_free(priv_key);
1780                         return 0;
1781                 }
1782         }
1783
1784         /* generate public and private key pair.
1785          *
1786          * if private key already set, generates only public key.
1787          *
1788          * if private key is not already set, then set it to random value
1789          * and update internal private key.
1790          */
1791         if (!DH_generate_key(dh_key)) {
1792                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1793                 return 0;
1794         }
1795
1796         if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) {
1797                 const BIGNUM *pub_key = NULL;
1798
1799                 OPENSSL_LOG(DEBUG, "%s:%d update public key\n",
1800                                 __func__, __LINE__);
1801
1802                 /* get the generated keys */
1803                 get_dh_pub_key(dh_key, &pub_key);
1804
1805                 /* output public key */
1806                 op->pub_key.length = BN_bn2bin(pub_key,
1807                                 op->pub_key.data);
1808         }
1809
1810         if (sess->u.dh.key_op &
1811                         (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) {
1812                 const BIGNUM *priv_key = NULL;
1813
1814                 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n",
1815                                 __func__, __LINE__);
1816
1817                 /* get the generated keys */
1818                 get_dh_priv_key(dh_key, &priv_key);
1819
1820                 /* provide generated private key back to user */
1821                 op->priv_key.length = BN_bn2bin(priv_key,
1822                                 op->priv_key.data);
1823         }
1824
1825         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1826
1827         return 0;
1828 }
1829
1830 /* process modinv operation */
1831 static int
1832 process_openssl_modinv_op(struct rte_crypto_op *cop,
1833                 struct openssl_asym_session *sess)
1834 {
1835         struct rte_crypto_asym_op *op = cop->asym;
1836         BIGNUM *base = BN_CTX_get(sess->u.m.ctx);
1837         BIGNUM *res = BN_CTX_get(sess->u.m.ctx);
1838
1839         if (unlikely(base == NULL || res == NULL)) {
1840                 BN_free(base);
1841                 BN_free(res);
1842                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1843                 return -1;
1844         }
1845
1846         base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
1847                         op->modinv.base.length, base);
1848
1849         if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) {
1850                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1851                 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data);
1852         } else {
1853                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1854         }
1855
1856         BN_clear(res);
1857         BN_clear(base);
1858
1859         return 0;
1860 }
1861
1862 /* process modexp operation */
1863 static int
1864 process_openssl_modexp_op(struct rte_crypto_op *cop,
1865                 struct openssl_asym_session *sess)
1866 {
1867         struct rte_crypto_asym_op *op = cop->asym;
1868         BIGNUM *base = BN_CTX_get(sess->u.e.ctx);
1869         BIGNUM *res = BN_CTX_get(sess->u.e.ctx);
1870
1871         if (unlikely(base == NULL || res == NULL)) {
1872                 BN_free(base);
1873                 BN_free(res);
1874                 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1875                 return -1;
1876         }
1877
1878         base = BN_bin2bn((const unsigned char *)op->modex.base.data,
1879                         op->modex.base.length, base);
1880
1881         if (BN_mod_exp(res, base, sess->u.e.exp,
1882                                 sess->u.e.mod, sess->u.e.ctx)) {
1883                 op->modex.result.length = BN_bn2bin(res, op->modex.result.data);
1884                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1885         } else {
1886                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1887         }
1888
1889         BN_clear(res);
1890         BN_clear(base);
1891
1892         return 0;
1893 }
1894
1895 /* process rsa operations */
1896 static int
1897 process_openssl_rsa_op(struct rte_crypto_op *cop,
1898                 struct openssl_asym_session *sess)
1899 {
1900         int ret = 0;
1901         struct rte_crypto_asym_op *op = cop->asym;
1902         RSA *rsa = sess->u.r.rsa;
1903         uint32_t pad = (op->rsa.pad);
1904         uint8_t *tmp;
1905
1906         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1907
1908         switch (pad) {
1909         case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
1910                 pad = RSA_PKCS1_PADDING;
1911                 break;
1912         case RTE_CRYPTO_RSA_PADDING_NONE:
1913                 pad = RSA_NO_PADDING;
1914                 break;
1915         default:
1916                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1917                 OPENSSL_LOG(ERR,
1918                                 "rsa pad type not supported %d\n", pad);
1919                 return 0;
1920         }
1921
1922         switch (op->rsa.op_type) {
1923         case RTE_CRYPTO_ASYM_OP_ENCRYPT:
1924                 ret = RSA_public_encrypt(op->rsa.message.length,
1925                                 op->rsa.message.data,
1926                                 op->rsa.cipher.data,
1927                                 rsa,
1928                                 pad);
1929
1930                 if (ret > 0)
1931                         op->rsa.cipher.length = ret;
1932                 OPENSSL_LOG(DEBUG,
1933                                 "length of encrypted text %d\n", ret);
1934                 break;
1935
1936         case RTE_CRYPTO_ASYM_OP_DECRYPT:
1937                 ret = RSA_private_decrypt(op->rsa.cipher.length,
1938                                 op->rsa.cipher.data,
1939                                 op->rsa.message.data,
1940                                 rsa,
1941                                 pad);
1942                 if (ret > 0)
1943                         op->rsa.message.length = ret;
1944                 break;
1945
1946         case RTE_CRYPTO_ASYM_OP_SIGN:
1947                 ret = RSA_private_encrypt(op->rsa.message.length,
1948                                 op->rsa.message.data,
1949                                 op->rsa.sign.data,
1950                                 rsa,
1951                                 pad);
1952                 if (ret > 0)
1953                         op->rsa.sign.length = ret;
1954                 break;
1955
1956         case RTE_CRYPTO_ASYM_OP_VERIFY:
1957                 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
1958                 if (tmp == NULL) {
1959                         OPENSSL_LOG(ERR, "Memory allocation failed");
1960                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1961                         break;
1962                 }
1963                 ret = RSA_public_decrypt(op->rsa.sign.length,
1964                                 op->rsa.sign.data,
1965                                 tmp,
1966                                 rsa,
1967                                 pad);
1968
1969                 OPENSSL_LOG(DEBUG,
1970                                 "Length of public_decrypt %d "
1971                                 "length of message %zd\n",
1972                                 ret, op->rsa.message.length);
1973                 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data,
1974                                 op->rsa.message.length))) {
1975                         OPENSSL_LOG(ERR, "RSA sign Verification failed");
1976                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1977                 }
1978                 rte_free(tmp);
1979                 break;
1980
1981         default:
1982                 /* allow ops with invalid args to be pushed to
1983                  * completion queue
1984                  */
1985                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1986                 break;
1987         }
1988
1989         if (ret < 0)
1990                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1991
1992         return 0;
1993 }
1994
1995 static int
1996 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1997                 struct openssl_asym_session *sess)
1998 {
1999         int retval = 0;
2000
2001         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2002
2003         switch (sess->xfrm_type) {
2004         case RTE_CRYPTO_ASYM_XFORM_RSA:
2005                 retval = process_openssl_rsa_op(op, sess);
2006                 break;
2007         case RTE_CRYPTO_ASYM_XFORM_MODEX:
2008                 retval = process_openssl_modexp_op(op, sess);
2009                 break;
2010         case RTE_CRYPTO_ASYM_XFORM_MODINV:
2011                 retval = process_openssl_modinv_op(op, sess);
2012                 break;
2013         case RTE_CRYPTO_ASYM_XFORM_DH:
2014                 retval = process_openssl_dh_op(op, sess);
2015                 break;
2016         case RTE_CRYPTO_ASYM_XFORM_DSA:
2017                 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
2018                         retval = process_openssl_dsa_sign_op(op, sess);
2019                 else if (op->asym->dsa.op_type ==
2020                                 RTE_CRYPTO_ASYM_OP_VERIFY)
2021                         retval =
2022                                 process_openssl_dsa_verify_op(op, sess);
2023                 else
2024                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2025                 break;
2026         default:
2027                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2028                 break;
2029         }
2030         if (!retval) {
2031                 /* op processed so push to completion queue as processed */
2032                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2033                 if (retval)
2034                         /* return error if failed to put in completion queue */
2035                         retval = -1;
2036         }
2037
2038         return retval;
2039 }
2040
2041 /** Process crypto operation for mbuf */
2042 static int
2043 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2044                 struct openssl_session *sess)
2045 {
2046         struct rte_mbuf *msrc, *mdst;
2047         int retval;
2048
2049         msrc = op->sym->m_src;
2050         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
2051
2052         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2053
2054         switch (sess->chain_order) {
2055         case OPENSSL_CHAIN_ONLY_CIPHER:
2056                 process_openssl_cipher_op(op, sess, msrc, mdst);
2057                 break;
2058         case OPENSSL_CHAIN_ONLY_AUTH:
2059                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2060                 break;
2061         case OPENSSL_CHAIN_CIPHER_AUTH:
2062                 process_openssl_cipher_op(op, sess, msrc, mdst);
2063                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
2064                 break;
2065         case OPENSSL_CHAIN_AUTH_CIPHER:
2066                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2067                 process_openssl_cipher_op(op, sess, msrc, mdst);
2068                 break;
2069         case OPENSSL_CHAIN_COMBINED:
2070                 process_openssl_combined_op(op, sess, msrc, mdst);
2071                 break;
2072         case OPENSSL_CHAIN_CIPHER_BPI:
2073                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
2074                 break;
2075         default:
2076                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
2077                 break;
2078         }
2079
2080         /* Free session if a session-less crypto op */
2081         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
2082                 openssl_reset_session(sess);
2083                 memset(sess, 0, sizeof(struct openssl_session));
2084                 memset(op->sym->session, 0,
2085                         rte_cryptodev_sym_get_existing_header_session_size(
2086                                 op->sym->session));
2087                 rte_mempool_put(qp->sess_mp_priv, sess);
2088                 rte_mempool_put(qp->sess_mp, op->sym->session);
2089                 op->sym->session = NULL;
2090         }
2091
2092         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
2093                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2094
2095         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
2096                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2097         else
2098                 retval = -1;
2099
2100         return retval;
2101 }
2102
2103 /*
2104  *------------------------------------------------------------------------------
2105  * PMD Framework
2106  *------------------------------------------------------------------------------
2107  */
2108
2109 /** Enqueue burst */
2110 static uint16_t
2111 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
2112                 uint16_t nb_ops)
2113 {
2114         void *sess;
2115         struct openssl_qp *qp = queue_pair;
2116         int i, retval;
2117
2118         for (i = 0; i < nb_ops; i++) {
2119                 sess = get_session(qp, ops[i]);
2120                 if (unlikely(sess == NULL))
2121                         goto enqueue_err;
2122
2123                 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
2124                         retval = process_op(qp, ops[i],
2125                                         (struct openssl_session *) sess);
2126                 else
2127                         retval = process_asym_op(qp, ops[i],
2128                                         (struct openssl_asym_session *) sess);
2129                 if (unlikely(retval < 0))
2130                         goto enqueue_err;
2131         }
2132
2133         qp->stats.enqueued_count += i;
2134         return i;
2135
2136 enqueue_err:
2137         qp->stats.enqueue_err_count++;
2138         return i;
2139 }
2140
2141 /** Dequeue burst */
2142 static uint16_t
2143 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
2144                 uint16_t nb_ops)
2145 {
2146         struct openssl_qp *qp = queue_pair;
2147
2148         unsigned int nb_dequeued = 0;
2149
2150         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
2151                         (void **)ops, nb_ops, NULL);
2152         qp->stats.dequeued_count += nb_dequeued;
2153
2154         return nb_dequeued;
2155 }
2156
2157 /** Create OPENSSL crypto device */
2158 static int
2159 cryptodev_openssl_create(const char *name,
2160                         struct rte_vdev_device *vdev,
2161                         struct rte_cryptodev_pmd_init_params *init_params)
2162 {
2163         struct rte_cryptodev *dev;
2164         struct openssl_private *internals;
2165
2166         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2167         if (dev == NULL) {
2168                 OPENSSL_LOG(ERR, "failed to create cryptodev vdev");
2169                 goto init_error;
2170         }
2171
2172         dev->driver_id = cryptodev_driver_id;
2173         dev->dev_ops = rte_openssl_pmd_ops;
2174
2175         /* register rx/tx burst functions for data path */
2176         dev->dequeue_burst = openssl_pmd_dequeue_burst;
2177         dev->enqueue_burst = openssl_pmd_enqueue_burst;
2178
2179         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2180                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2181                         RTE_CRYPTODEV_FF_CPU_AESNI |
2182                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2183                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2184                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2185                         RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
2186                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP |
2187                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT;
2188
2189         internals = dev->data->dev_private;
2190
2191         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
2192
2193         return 0;
2194
2195 init_error:
2196         OPENSSL_LOG(ERR, "driver %s: create failed",
2197                         init_params->name);
2198
2199         cryptodev_openssl_remove(vdev);
2200         return -EFAULT;
2201 }
2202
2203 /** Initialise OPENSSL crypto device */
2204 static int
2205 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
2206 {
2207         struct rte_cryptodev_pmd_init_params init_params = {
2208                 "",
2209                 sizeof(struct openssl_private),
2210                 rte_socket_id(),
2211                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2212         };
2213         const char *name;
2214         const char *input_args;
2215
2216         name = rte_vdev_device_name(vdev);
2217         if (name == NULL)
2218                 return -EINVAL;
2219         input_args = rte_vdev_device_args(vdev);
2220
2221         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2222
2223         return cryptodev_openssl_create(name, vdev, &init_params);
2224 }
2225
2226 /** Uninitialise OPENSSL crypto device */
2227 static int
2228 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
2229 {
2230         struct rte_cryptodev *cryptodev;
2231         const char *name;
2232
2233         name = rte_vdev_device_name(vdev);
2234         if (name == NULL)
2235                 return -EINVAL;
2236
2237         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2238         if (cryptodev == NULL)
2239                 return -ENODEV;
2240
2241         return rte_cryptodev_pmd_destroy(cryptodev);
2242 }
2243
2244 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
2245         .probe = cryptodev_openssl_probe,
2246         .remove = cryptodev_openssl_remove
2247 };
2248
2249 static struct cryptodev_driver openssl_crypto_drv;
2250
2251 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
2252         cryptodev_openssl_pmd_drv);
2253 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
2254         "max_nb_queue_pairs=<int> "
2255         "socket_id=<int>");
2256 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
2257                 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);
2258
2259 RTE_INIT(openssl_init_log)
2260 {
2261         openssl_logtype_driver = rte_log_register("pmd.crypto.openssl");
2262 }