drivers/crypto/openssl/rte_openssl_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_hexdump.h>
   7 #include <rte_cryptodev.h>
   8 #include <rte_cryptodev_pmd.h>
   9 #include <rte_bus_vdev.h>
  10 #include <rte_malloc.h>
  11 #include <rte_cpuflags.h>
  12
  13 #include <openssl/hmac.h>
  14 #include <openssl/evp.h>
  15
  16 #include "rte_openssl_pmd_private.h"
  17
  18 #define DES_BLOCK_SIZE 8
  19
  20 static uint8_t cryptodev_driver_id;
  21
  22 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  23 static HMAC_CTX *HMAC_CTX_new(void)
  24 {
  25         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  26
  27         if (ctx != NULL)
  28                 HMAC_CTX_init(ctx);
  29         return ctx;
  30 }
  31
  32 static void HMAC_CTX_free(HMAC_CTX *ctx)
  33 {
  34         if (ctx != NULL) {
  35                 HMAC_CTX_cleanup(ctx);
  36                 OPENSSL_free(ctx);
  37         }
  38 }
  39 #endif
  40
  41 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  42
  43 /*----------------------------------------------------------------------------*/
  44
  45 /**
  46  * Increment counter by 1
  47  * Counter is 64 bit array, big-endian
  48  */
  49 static void
  50 ctr_inc(uint8_t *ctr)
  51 {
  52         uint64_t *ctr64 = (uint64_t *)ctr;
  53
  54         *ctr64 = __builtin_bswap64(*ctr64);
  55         (*ctr64)++;
  56         *ctr64 = __builtin_bswap64(*ctr64);
  57 }
  58
  59 /*
  60  *------------------------------------------------------------------------------
  61  * Session Prepare
  62  *------------------------------------------------------------------------------
  63  */
  64
  65 /** Get xform chain order */
  66 static enum openssl_chain_order
  67 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
  68 {
  69         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
  70
  71         if (xform != NULL) {
  72                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  73                         if (xform->next == NULL)
  74                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
  75                         else if (xform->next->type ==
  76                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
  77                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
  78                 }
  79                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  80                         if (xform->next == NULL)
  81                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
  82                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  83                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
  84                 }
  85                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
  86                         res = OPENSSL_CHAIN_COMBINED;
  87         }
  88
  89         return res;
  90 }
  91
  92 /** Get session cipher key from input cipher key */
  93 static void
  94 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
  95 {
  96         memcpy(session_key, input_key, keylen);
  97 }
  98
  99 /** Get key ede 24 bytes standard from input key */
 100 static int
 101 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
 102 {
 103         int res = 0;
 104
 105         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 106         switch (keylen) {
 107         case 24:
 108                 memcpy(key_ede, key, 24);
 109                 break;
 110         case 16:
 111                 /* K3 = K1 */
 112                 memcpy(key_ede, key, 16);
 113                 memcpy(key_ede + 16, key, 8);
 114                 break;
 115         case 8:
 116                 /* K1 = K2 = K3 (DES compatibility) */
 117                 memcpy(key_ede, key, 8);
 118                 memcpy(key_ede + 8, key, 8);
 119                 memcpy(key_ede + 16, key, 8);
 120                 break;
 121         default:
 122                 OPENSSL_LOG_ERR("Unsupported key size");
 123                 res = -EINVAL;
 124         }
 125
 126         return res;
 127 }
 128
 129 /** Get adequate openssl function for input cipher algorithm */
 130 static uint8_t
 131 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 132                 const EVP_CIPHER **algo)
 133 {
 134         int res = 0;
 135
 136         if (algo != NULL) {
 137                 switch (sess_algo) {
 138                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 139                         switch (keylen) {
 140                         case 16:
 141                                 *algo = EVP_des_ede_cbc();
 142                                 break;
 143                         case 24:
 144                                 *algo = EVP_des_ede3_cbc();
 145                                 break;
 146                         default:
 147                                 res = -EINVAL;
 148                         }
 149                         break;
 150                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 151                         break;
 152                 case RTE_CRYPTO_CIPHER_AES_CBC:
 153                         switch (keylen) {
 154                         case 16:
 155                                 *algo = EVP_aes_128_cbc();
 156                                 break;
 157                         case 24:
 158                                 *algo = EVP_aes_192_cbc();
 159                                 break;
 160                         case 32:
 161                                 *algo = EVP_aes_256_cbc();
 162                                 break;
 163                         default:
 164                                 res = -EINVAL;
 165                         }
 166                         break;
 167                 case RTE_CRYPTO_CIPHER_AES_CTR:
 168                         switch (keylen) {
 169                         case 16:
 170                                 *algo = EVP_aes_128_ctr();
 171                                 break;
 172                         case 24:
 173                                 *algo = EVP_aes_192_ctr();
 174                                 break;
 175                         case 32:
 176                                 *algo = EVP_aes_256_ctr();
 177                                 break;
 178                         default:
 179                                 res = -EINVAL;
 180                         }
 181                         break;
 182                 default:
 183                         res = -EINVAL;
 184                         break;
 185                 }
 186         } else {
 187                 res = -EINVAL;
 188         }
 189
 190         return res;
 191 }
 192
 193 /** Get adequate openssl function for input auth algorithm */
 194 static uint8_t
 195 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 196                 const EVP_MD **algo)
 197 {
 198         int res = 0;
 199
 200         if (algo != NULL) {
 201                 switch (sessalgo) {
 202                 case RTE_CRYPTO_AUTH_MD5:
 203                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 204                         *algo = EVP_md5();
 205                         break;
 206                 case RTE_CRYPTO_AUTH_SHA1:
 207                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 208                         *algo = EVP_sha1();
 209                         break;
 210                 case RTE_CRYPTO_AUTH_SHA224:
 211                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 212                         *algo = EVP_sha224();
 213                         break;
 214                 case RTE_CRYPTO_AUTH_SHA256:
 215                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 216                         *algo = EVP_sha256();
 217                         break;
 218                 case RTE_CRYPTO_AUTH_SHA384:
 219                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 220                         *algo = EVP_sha384();
 221                         break;
 222                 case RTE_CRYPTO_AUTH_SHA512:
 223                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 224                         *algo = EVP_sha512();
 225                         break;
 226                 default:
 227                         res = -EINVAL;
 228                         break;
 229                 }
 230         } else {
 231                 res = -EINVAL;
 232         }
 233
 234         return res;
 235 }
 236
 237 /** Get adequate openssl function for input cipher algorithm */
 238 static uint8_t
 239 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 240                 const EVP_CIPHER **algo)
 241 {
 242         int res = 0;
 243
 244         if (algo != NULL) {
 245                 switch (sess_algo) {
 246                 case RTE_CRYPTO_AEAD_AES_GCM:
 247                         switch (keylen) {
 248                         case 16:
 249                                 *algo = EVP_aes_128_gcm();
 250                                 break;
 251                         case 24:
 252                                 *algo = EVP_aes_192_gcm();
 253                                 break;
 254                         case 32:
 255                                 *algo = EVP_aes_256_gcm();
 256                                 break;
 257                         default:
 258                                 res = -EINVAL;
 259                         }
 260                         break;
 261                 case RTE_CRYPTO_AEAD_AES_CCM:
 262                         switch (keylen) {
 263                         case 16:
 264                                 *algo = EVP_aes_128_ccm();
 265                                 break;
 266                         case 24:
 267                                 *algo = EVP_aes_192_ccm();
 268                                 break;
 269                         case 32:
 270                                 *algo = EVP_aes_256_ccm();
 271                                 break;
 272                         default:
 273                                 res = -EINVAL;
 274                         }
 275                         break;
 276                 default:
 277                         res = -EINVAL;
 278                         break;
 279                 }
 280         } else {
 281                 res = -EINVAL;
 282         }
 283
 284         return res;
 285 }
 286
 287 /* Set session AEAD encryption parameters */
 288 static int
 289 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
 290                 enum rte_crypto_aead_algorithm algo,
 291                 uint8_t tag_len, uint8_t *key)
 292 {
 293         int iv_type = 0;
 294         unsigned int do_ccm;
 295
 296         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 297         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 298
 299         /* Select AEAD algo */
 300         switch (algo) {
 301         case RTE_CRYPTO_AEAD_AES_GCM:
 302                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 303                 if (tag_len != 16)
 304                         return -EINVAL;
 305                 do_ccm = 0;
 306                 break;
 307         case RTE_CRYPTO_AEAD_AES_CCM:
 308                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 309                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 310                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 311                         return -EINVAL;
 312                 do_ccm = 1;
 313                 break;
 314         default:
 315                 return -ENOTSUP;
 316         }
 317
 318         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 319         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 320
 321         if (get_aead_algo(algo, sess->cipher.key.length,
 322                         &sess->cipher.evp_algo) != 0)
 323                 return -EINVAL;
 324
 325         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 326
 327         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 328
 329         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 330                         NULL, NULL, NULL) <= 0)
 331                 return -EINVAL;
 332
 333         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
 334                         NULL) <= 0)
 335                 return -EINVAL;
 336
 337         if (do_ccm)
 338                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 339                                 tag_len, NULL);
 340
 341         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 342                 return -EINVAL;
 343
 344         return 0;
 345 }
 346
 347 /* Set session AEAD decryption parameters */
 348 static int
 349 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
 350                 enum rte_crypto_aead_algorithm algo,
 351                 uint8_t tag_len, uint8_t *key)
 352 {
 353         int iv_type = 0;
 354         unsigned int do_ccm = 0;
 355
 356         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 357         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 358
 359         /* Select AEAD algo */
 360         switch (algo) {
 361         case RTE_CRYPTO_AEAD_AES_GCM:
 362                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 363                 if (tag_len != 16)
 364                         return -EINVAL;
 365                 break;
 366         case RTE_CRYPTO_AEAD_AES_CCM:
 367                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 368                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 369                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 370                         return -EINVAL;
 371                 do_ccm = 1;
 372                 break;
 373         default:
 374                 return -ENOTSUP;
 375         }
 376
 377         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 378         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 379
 380         if (get_aead_algo(algo, sess->cipher.key.length,
 381                         &sess->cipher.evp_algo) != 0)
 382                 return -EINVAL;
 383
 384         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 385
 386         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 387
 388         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 389                         NULL, NULL, NULL) <= 0)
 390                 return -EINVAL;
 391
 392         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
 393                         sess->iv.length, NULL) <= 0)
 394                 return -EINVAL;
 395
 396         if (do_ccm)
 397                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 398                                 tag_len, NULL);
 399
 400         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 401                 return -EINVAL;
 402
 403         return 0;
 404 }
 405
 406 /** Set session cipher parameters */
 407 static int
 408 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 409                 const struct rte_crypto_sym_xform *xform)
 410 {
 411         /* Select cipher direction */
 412         sess->cipher.direction = xform->cipher.op;
 413         /* Select cipher key */
 414         sess->cipher.key.length = xform->cipher.key.length;
 415
 416         /* Set IV parameters */
 417         sess->iv.offset = xform->cipher.iv.offset;
 418         sess->iv.length = xform->cipher.iv.length;
 419
 420         /* Select cipher algo */
 421         switch (xform->cipher.algo) {
 422         case RTE_CRYPTO_CIPHER_3DES_CBC:
 423         case RTE_CRYPTO_CIPHER_AES_CBC:
 424         case RTE_CRYPTO_CIPHER_AES_CTR:
 425                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 426                 sess->cipher.algo = xform->cipher.algo;
 427                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 428
 429                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 430                                 &sess->cipher.evp_algo) != 0)
 431                         return -EINVAL;
 432
 433                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 434                         sess->cipher.key.data);
 435                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 436                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 437                                         sess->cipher.evp_algo,
 438                                         NULL, xform->cipher.key.data,
 439                                         NULL) != 1) {
 440                                 return -EINVAL;
 441                         }
 442                 } else if (sess->cipher.direction ==
 443                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 444                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 445                                         sess->cipher.evp_algo,
 446                                         NULL, xform->cipher.key.data,
 447                                         NULL) != 1) {
 448                                 return -EINVAL;
 449                         }
 450                 }
 451
 452                 break;
 453
 454         case RTE_CRYPTO_CIPHER_3DES_CTR:
 455                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 456                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 457
 458                 if (get_cipher_key_ede(xform->cipher.key.data,
 459                                 sess->cipher.key.length,
 460                                 sess->cipher.key.data) != 0)
 461                         return -EINVAL;
 462                 break;
 463
 464         case RTE_CRYPTO_CIPHER_DES_CBC:
 465                 sess->cipher.algo = xform->cipher.algo;
 466                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 467                 sess->cipher.evp_algo = EVP_des_cbc();
 468
 469                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 470                         sess->cipher.key.data);
 471                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 472                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 473                                         sess->cipher.evp_algo,
 474                                         NULL, xform->cipher.key.data,
 475                                         NULL) != 1) {
 476                                 return -EINVAL;
 477                         }
 478                 } else if (sess->cipher.direction ==
 479                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 480                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 481                                         sess->cipher.evp_algo,
 482                                         NULL, xform->cipher.key.data,
 483                                         NULL) != 1) {
 484                                 return -EINVAL;
 485                         }
 486                 }
 487
 488                 break;
 489
 490         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 491                 sess->cipher.algo = xform->cipher.algo;
 492                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 493                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 494                 sess->cipher.evp_algo = EVP_des_cbc();
 495
 496                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 497                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 498                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 499                                 NULL, xform->cipher.key.data, 0) != 1)
 500                         return -EINVAL;
 501
 502                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 503                         sess->cipher.key.data);
 504                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 505                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 506                                         sess->cipher.evp_algo,
 507                                         NULL, xform->cipher.key.data,
 508                                         NULL) != 1) {
 509                                 return -EINVAL;
 510                         }
 511                 } else if (sess->cipher.direction ==
 512                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 513                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 514                                         sess->cipher.evp_algo,
 515                                         NULL, xform->cipher.key.data,
 516                                         NULL) != 1) {
 517                                 return -EINVAL;
 518                         }
 519                 }
 520
 521                 break;
 522         default:
 523                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 524                 return -ENOTSUP;
 525         }
 526
 527         return 0;
 528 }
 529
 530 /* Set session auth parameters */
 531 static int
 532 openssl_set_session_auth_parameters(struct openssl_session *sess,
 533                 const struct rte_crypto_sym_xform *xform)
 534 {
 535         /* Select auth generate/verify */
 536         sess->auth.operation = xform->auth.op;
 537         sess->auth.algo = xform->auth.algo;
 538
 539         sess->auth.digest_length = xform->auth.digest_length;
 540
 541         /* Select auth algo */
 542         switch (xform->auth.algo) {
 543         case RTE_CRYPTO_AUTH_AES_GMAC:
 544                 /*
 545                  * OpenSSL requires GMAC to be a GCM operation
 546                  * with no cipher data length
 547                  */
 548                 sess->cipher.key.length = xform->auth.key.length;
 549
 550                 /* Set IV parameters */
 551                 sess->iv.offset = xform->auth.iv.offset;
 552                 sess->iv.length = xform->auth.iv.length;
 553
 554                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 555                         return openssl_set_sess_aead_enc_param(sess,
 556                                                 RTE_CRYPTO_AEAD_AES_GCM,
 557                                                 xform->auth.digest_length,
 558                                                 xform->auth.key.data);
 559                 else
 560                         return openssl_set_sess_aead_dec_param(sess,
 561                                                 RTE_CRYPTO_AEAD_AES_GCM,
 562                                                 xform->auth.digest_length,
 563                                                 xform->auth.key.data);
 564                 break;
 565
 566         case RTE_CRYPTO_AUTH_MD5:
 567         case RTE_CRYPTO_AUTH_SHA1:
 568         case RTE_CRYPTO_AUTH_SHA224:
 569         case RTE_CRYPTO_AUTH_SHA256:
 570         case RTE_CRYPTO_AUTH_SHA384:
 571         case RTE_CRYPTO_AUTH_SHA512:
 572                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 573                 if (get_auth_algo(xform->auth.algo,
 574                                 &sess->auth.auth.evp_algo) != 0)
 575                         return -EINVAL;
 576                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 577                 break;
 578
 579         case RTE_CRYPTO_AUTH_MD5_HMAC:
 580         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 581         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 582         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 583         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 584         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 585                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 586                 sess->auth.hmac.ctx = HMAC_CTX_new();
 587                 if (get_auth_algo(xform->auth.algo,
 588                                 &sess->auth.hmac.evp_algo) != 0)
 589                         return -EINVAL;
 590
 591                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 592                                 xform->auth.key.data,
 593                                 xform->auth.key.length,
 594                                 sess->auth.hmac.evp_algo, NULL) != 1)
 595                         return -EINVAL;
 596                 break;
 597
 598         default:
 599                 return -ENOTSUP;
 600         }
 601
 602         return 0;
 603 }
 604
 605 /* Set session AEAD parameters */
 606 static int
 607 openssl_set_session_aead_parameters(struct openssl_session *sess,
 608                 const struct rte_crypto_sym_xform *xform)
 609 {
 610         /* Select cipher key */
 611         sess->cipher.key.length = xform->aead.key.length;
 612
 613         /* Set IV parameters */
 614         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 615                 /*
 616                  * For AES-CCM, the actual IV is placed
 617                  * one byte after the start of the IV field,
 618                  * according to the API.
 619                  */
 620                 sess->iv.offset = xform->aead.iv.offset + 1;
 621         else
 622                 sess->iv.offset = xform->aead.iv.offset;
 623
 624         sess->iv.length = xform->aead.iv.length;
 625
 626         sess->auth.aad_length = xform->aead.aad_length;
 627         sess->auth.digest_length = xform->aead.digest_length;
 628
 629         sess->aead_algo = xform->aead.algo;
 630         /* Select cipher direction */
 631         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 632                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
 633                                 xform->aead.digest_length, xform->aead.key.data);
 634         else
 635                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
 636                                 xform->aead.digest_length, xform->aead.key.data);
 637 }
 638
 639 /** Parse crypto xform chain and set private session parameters */
 640 int
 641 openssl_set_session_parameters(struct openssl_session *sess,
 642                 const struct rte_crypto_sym_xform *xform)
 643 {
 644         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 645         const struct rte_crypto_sym_xform *auth_xform = NULL;
 646         const struct rte_crypto_sym_xform *aead_xform = NULL;
 647         int ret;
 648
 649         sess->chain_order = openssl_get_chain_order(xform);
 650         switch (sess->chain_order) {
 651         case OPENSSL_CHAIN_ONLY_CIPHER:
 652                 cipher_xform = xform;
 653                 break;
 654         case OPENSSL_CHAIN_ONLY_AUTH:
 655                 auth_xform = xform;
 656                 break;
 657         case OPENSSL_CHAIN_CIPHER_AUTH:
 658                 cipher_xform = xform;
 659                 auth_xform = xform->next;
 660                 break;
 661         case OPENSSL_CHAIN_AUTH_CIPHER:
 662                 auth_xform = xform;
 663                 cipher_xform = xform->next;
 664                 break;
 665         case OPENSSL_CHAIN_COMBINED:
 666                 aead_xform = xform;
 667                 break;
 668         default:
 669                 return -EINVAL;
 670         }
 671
 672         /* Default IV length = 0 */
 673         sess->iv.length = 0;
 674
 675         /* cipher_xform must be check before auth_xform */
 676         if (cipher_xform) {
 677                 ret = openssl_set_session_cipher_parameters(
 678                                 sess, cipher_xform);
 679                 if (ret != 0) {
 680                         OPENSSL_LOG_ERR(
 681                                 "Invalid/unsupported cipher parameters");
 682                         return ret;
 683                 }
 684         }
 685
 686         if (auth_xform) {
 687                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 688                 if (ret != 0) {
 689                         OPENSSL_LOG_ERR(
 690                                 "Invalid/unsupported auth parameters");
 691                         return ret;
 692                 }
 693         }
 694
 695         if (aead_xform) {
 696                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 697                 if (ret != 0) {
 698                         OPENSSL_LOG_ERR(
 699                                 "Invalid/unsupported AEAD parameters");
 700                         return ret;
 701                 }
 702         }
 703
 704         return 0;
 705 }
 706
 707 /** Reset private session parameters */
 708 void
 709 openssl_reset_session(struct openssl_session *sess)
 710 {
 711         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 712
 713         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 714                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 715
 716         switch (sess->auth.mode) {
 717         case OPENSSL_AUTH_AS_AUTH:
 718                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 719                 break;
 720         case OPENSSL_AUTH_AS_HMAC:
 721                 EVP_PKEY_free(sess->auth.hmac.pkey);
 722                 HMAC_CTX_free(sess->auth.hmac.ctx);
 723                 break;
 724         default:
 725                 break;
 726         }
 727 }
 728
 729 /** Provide session for operation */
 730 static struct openssl_session *
 731 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 732 {
 733         struct openssl_session *sess = NULL;
 734
 735         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 736                 /* get existing session */
 737                 if (likely(op->sym->session != NULL))
 738                         sess = (struct openssl_session *)
 739                                         get_session_private_data(
 740                                         op->sym->session,
 741                                         cryptodev_driver_id);
 742         } else {
 743                 /* provide internal session */
 744                 void *_sess = NULL;
 745                 void *_sess_private_data = NULL;
 746
 747                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 748                         return NULL;
 749
 750                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 751                         return NULL;
 752
 753                 sess = (struct openssl_session *)_sess_private_data;
 754
 755                 if (unlikely(openssl_set_session_parameters(sess,
 756                                 op->sym->xform) != 0)) {
 757                         rte_mempool_put(qp->sess_mp, _sess);
 758                         rte_mempool_put(qp->sess_mp, _sess_private_data);
 759                         sess = NULL;
 760                 }
 761                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 762                 set_session_private_data(op->sym->session, cryptodev_driver_id,
 763                         _sess_private_data);
 764         }
 765
 766         if (sess == NULL)
 767                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 768
 769         return sess;
 770 }
 771
 772 /*
 773  *------------------------------------------------------------------------------
 774  * Process Operations
 775  *------------------------------------------------------------------------------
 776  */
 777 static inline int
 778 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 779                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 780 {
 781         struct rte_mbuf *m;
 782         int dstlen;
 783         int l, n = srclen;
 784         uint8_t *src;
 785
 786         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 787                         m = m->next)
 788                 offset -= rte_pktmbuf_data_len(m);
 789
 790         if (m == 0)
 791                 return -1;
 792
 793         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 794
 795         l = rte_pktmbuf_data_len(m) - offset;
 796         if (srclen <= l) {
 797                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 798                         return -1;
 799                 *dst += l;
 800                 return 0;
 801         }
 802
 803         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 804                 return -1;
 805
 806         *dst += dstlen;
 807         n -= l;
 808
 809         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 810                 src = rte_pktmbuf_mtod(m, uint8_t *);
 811                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 812                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 813                         return -1;
 814                 *dst += dstlen;
 815                 n -= l;
 816         }
 817
 818         return 0;
 819 }
 820
 821 static inline int
 822 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 823                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 824 {
 825         struct rte_mbuf *m;
 826         int dstlen;
 827         int l, n = srclen;
 828         uint8_t *src;
 829
 830         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 831                         m = m->next)
 832                 offset -= rte_pktmbuf_data_len(m);
 833
 834         if (m == 0)
 835                 return -1;
 836
 837         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 838
 839         l = rte_pktmbuf_data_len(m) - offset;
 840         if (srclen <= l) {
 841                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 842                         return -1;
 843                 *dst += l;
 844                 return 0;
 845         }
 846
 847         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 848                 return -1;
 849
 850         *dst += dstlen;
 851         n -= l;
 852
 853         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 854                 src = rte_pktmbuf_mtod(m, uint8_t *);
 855                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 856                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 857                         return -1;
 858                 *dst += dstlen;
 859                 n -= l;
 860         }
 861
 862         return 0;
 863 }
 864
 865 /** Process standard openssl cipher encryption */
 866 static int
 867 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 868                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 869 {
 870         int totlen;
 871
 872         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 873                 goto process_cipher_encrypt_err;
 874
 875         EVP_CIPHER_CTX_set_padding(ctx, 0);
 876
 877         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 878                         srclen, ctx))
 879                 goto process_cipher_encrypt_err;
 880
 881         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
 882                 goto process_cipher_encrypt_err;
 883
 884         return 0;
 885
 886 process_cipher_encrypt_err:
 887         OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
 888         return -EINVAL;
 889 }
 890
 891 /** Process standard openssl cipher encryption */
 892 static int
 893 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 894                 uint8_t *iv, int srclen,
 895                 EVP_CIPHER_CTX *ctx)
 896 {
 897         uint8_t i;
 898         uint8_t encrypted_iv[DES_BLOCK_SIZE];
 899         int encrypted_ivlen;
 900
 901         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
 902                         iv, DES_BLOCK_SIZE) <= 0)
 903                 goto process_cipher_encrypt_err;
 904
 905         for (i = 0; i < srclen; i++)
 906                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
 907
 908         return 0;
 909
 910 process_cipher_encrypt_err:
 911         OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
 912         return -EINVAL;
 913 }
 914 /** Process standard openssl cipher decryption */
 915 static int
 916 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 917                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 918 {
 919         int totlen;
 920
 921         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 922                 goto process_cipher_decrypt_err;
 923
 924         EVP_CIPHER_CTX_set_padding(ctx, 0);
 925
 926         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 927                         srclen, ctx))
 928                 goto process_cipher_decrypt_err;
 929
 930         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
 931                 goto process_cipher_decrypt_err;
 932         return 0;
 933
 934 process_cipher_decrypt_err:
 935         OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
 936         return -EINVAL;
 937 }
 938
 939 /** Process cipher des 3 ctr encryption, decryption algorithm */
 940 static int
 941 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
 942                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 943                 EVP_CIPHER_CTX *ctx)
 944 {
 945         uint8_t ebuf[8], ctr[8];
 946         int unused, n;
 947         struct rte_mbuf *m;
 948         uint8_t *src;
 949         int l;
 950
 951         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 952                         m = m->next)
 953                 offset -= rte_pktmbuf_data_len(m);
 954
 955         if (m == 0)
 956                 goto process_cipher_des3ctr_err;
 957
 958         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 959         l = rte_pktmbuf_data_len(m) - offset;
 960
 961         /* We use 3DES encryption also for decryption.
 962          * IV is not important for 3DES ecb
 963          */
 964         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
 965                 goto process_cipher_des3ctr_err;
 966
 967         memcpy(ctr, iv, 8);
 968
 969         for (n = 0; n < srclen; n++) {
 970                 if (n % 8 == 0) {
 971                         if (EVP_EncryptUpdate(ctx,
 972                                         (unsigned char *)&ebuf, &unused,
 973                                         (const unsigned char *)&ctr, 8) <= 0)
 974                                 goto process_cipher_des3ctr_err;
 975                         ctr_inc(ctr);
 976                 }
 977                 dst[n] = *(src++) ^ ebuf[n % 8];
 978
 979                 l--;
 980                 if (!l) {
 981                         m = m->next;
 982                         if (m) {
 983                                 src = rte_pktmbuf_mtod(m, uint8_t *);
 984                                 l = rte_pktmbuf_data_len(m);
 985                         }
 986                 }
 987         }
 988
 989         return 0;
 990
 991 process_cipher_des3ctr_err:
 992         OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
 993         return -EINVAL;
 994 }
 995
 996 /** Process AES-GCM encrypt algorithm */
 997 static int
 998 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 999                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1000                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1001 {
1002         int len = 0, unused = 0;
1003         uint8_t empty[] = {};
1004
1005         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1006                 goto process_auth_encryption_gcm_err;
1007
1008         if (aadlen > 0)
1009                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1010                         goto process_auth_encryption_gcm_err;
1011
1012         if (srclen > 0)
1013                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1014                                 srclen, ctx))
1015                         goto process_auth_encryption_gcm_err;
1016
1017         /* Workaround open ssl bug in version less then 1.0.1f */
1018         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1019                 goto process_auth_encryption_gcm_err;
1020
1021         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1022                 goto process_auth_encryption_gcm_err;
1023
1024         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1025                 goto process_auth_encryption_gcm_err;
1026
1027         return 0;
1028
1029 process_auth_encryption_gcm_err:
1030         OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
1031         return -EINVAL;
1032 }
1033
1034 /** Process AES-CCM encrypt algorithm */
1035 static int
1036 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1037                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1038                 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1039 {
1040         int len = 0;
1041
1042         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1043                 goto process_auth_encryption_ccm_err;
1044
1045         if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1046                 goto process_auth_encryption_ccm_err;
1047
1048         if (aadlen > 0)
1049                 /*
1050                  * For AES-CCM, the actual AAD is placed
1051                  * 18 bytes after the start of the AAD field,
1052                  * according to the API.
1053                  */
1054                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1055                         goto process_auth_encryption_ccm_err;
1056
1057         if (srclen > 0)
1058                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1059                                 srclen, ctx))
1060                         goto process_auth_encryption_ccm_err;
1061
1062         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1063                 goto process_auth_encryption_ccm_err;
1064
1065         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1066                 goto process_auth_encryption_ccm_err;
1067
1068         return 0;
1069
1070 process_auth_encryption_ccm_err:
1071         OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed");
1072         return -EINVAL;
1073 }
1074
1075 /** Process AES-GCM decrypt algorithm */
1076 static int
1077 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1078                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1079                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1080 {
1081         int len = 0, unused = 0;
1082         uint8_t empty[] = {};
1083
1084         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1085                 goto process_auth_decryption_gcm_err;
1086
1087         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1088                 goto process_auth_decryption_gcm_err;
1089
1090         if (aadlen > 0)
1091                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1092                         goto process_auth_decryption_gcm_err;
1093
1094         if (srclen > 0)
1095                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1096                                 srclen, ctx))
1097                         goto process_auth_decryption_gcm_err;
1098
1099         /* Workaround open ssl bug in version less then 1.0.1f */
1100         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1101                 goto process_auth_decryption_gcm_err;
1102
1103         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1104                 return -EFAULT;
1105
1106         return 0;
1107
1108 process_auth_decryption_gcm_err:
1109         OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed");
1110         return -EINVAL;
1111 }
1112
1113 /** Process AES-CCM decrypt algorithm */
1114 static int
1115 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1116                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1117                 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1118                 EVP_CIPHER_CTX *ctx)
1119 {
1120         int len = 0;
1121
1122         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1123                 goto process_auth_decryption_ccm_err;
1124
1125         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1126                 goto process_auth_decryption_ccm_err;
1127
1128         if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1129                 goto process_auth_decryption_ccm_err;
1130
1131         if (aadlen > 0)
1132                 /*
1133                  * For AES-CCM, the actual AAD is placed
1134                  * 18 bytes after the start of the AAD field,
1135                  * according to the API.
1136                  */
1137                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1138                         goto process_auth_decryption_ccm_err;
1139
1140         if (srclen > 0)
1141                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1142                                 srclen, ctx))
1143                         return -EFAULT;
1144
1145         return 0;
1146
1147 process_auth_decryption_ccm_err:
1148         OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed");
1149         return -EINVAL;
1150 }
1151
1152 /** Process standard openssl auth algorithms */
1153 static int
1154 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1155                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1156                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1157 {
1158         size_t dstlen;
1159         struct rte_mbuf *m;
1160         int l, n = srclen;
1161         uint8_t *src;
1162
1163         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1164                         m = m->next)
1165                 offset -= rte_pktmbuf_data_len(m);
1166
1167         if (m == 0)
1168                 goto process_auth_err;
1169
1170         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1171                 goto process_auth_err;
1172
1173         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1174
1175         l = rte_pktmbuf_data_len(m) - offset;
1176         if (srclen <= l) {
1177                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1178                         goto process_auth_err;
1179                 goto process_auth_final;
1180         }
1181
1182         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1183                 goto process_auth_err;
1184
1185         n -= l;
1186
1187         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1188                 src = rte_pktmbuf_mtod(m, uint8_t *);
1189                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1190                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1191                         goto process_auth_err;
1192                 n -= l;
1193         }
1194
1195 process_auth_final:
1196         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1197                 goto process_auth_err;
1198         return 0;
1199
1200 process_auth_err:
1201         OPENSSL_LOG_ERR("Process openssl auth failed");
1202         return -EINVAL;
1203 }
1204
1205 /** Process standard openssl auth algorithms with hmac */
1206 static int
1207 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1208                 int srclen, HMAC_CTX *ctx)
1209 {
1210         unsigned int dstlen;
1211         struct rte_mbuf *m;
1212         int l, n = srclen;
1213         uint8_t *src;
1214
1215         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1216                         m = m->next)
1217                 offset -= rte_pktmbuf_data_len(m);
1218
1219         if (m == 0)
1220                 goto process_auth_err;
1221
1222         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1223
1224         l = rte_pktmbuf_data_len(m) - offset;
1225         if (srclen <= l) {
1226                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1227                         goto process_auth_err;
1228                 goto process_auth_final;
1229         }
1230
1231         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1232                 goto process_auth_err;
1233
1234         n -= l;
1235
1236         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1237                 src = rte_pktmbuf_mtod(m, uint8_t *);
1238                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1239                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1240                         goto process_auth_err;
1241                 n -= l;
1242         }
1243
1244 process_auth_final:
1245         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1246                 goto process_auth_err;
1247
1248         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1249                 goto process_auth_err;
1250
1251         return 0;
1252
1253 process_auth_err:
1254         OPENSSL_LOG_ERR("Process openssl auth failed");
1255         return -EINVAL;
1256 }
1257
1258 /*----------------------------------------------------------------------------*/
1259
1260 /** Process auth/cipher combined operation */
1261 static void
1262 process_openssl_combined_op
1263                 (struct rte_crypto_op *op, struct openssl_session *sess,
1264                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1265 {
1266         /* cipher */
1267         uint8_t *dst = NULL, *iv, *tag, *aad;
1268         int srclen, aadlen, status = -1;
1269         uint32_t offset;
1270         uint8_t taglen;
1271
1272         /*
1273          * Segmented destination buffer is not supported for
1274          * encryption/decryption
1275          */
1276         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1277                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1278                 return;
1279         }
1280
1281         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1282                         sess->iv.offset);
1283         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1284                 srclen = 0;
1285                 offset = op->sym->auth.data.offset;
1286                 aadlen = op->sym->auth.data.length;
1287                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1288                                 op->sym->auth.data.offset);
1289                 tag = op->sym->auth.digest.data;
1290                 if (tag == NULL)
1291                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1292                                 offset + aadlen);
1293         } else {
1294                 srclen = op->sym->aead.data.length;
1295                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1296                                 op->sym->aead.data.offset);
1297                 offset = op->sym->aead.data.offset;
1298                 aad = op->sym->aead.aad.data;
1299                 aadlen = sess->auth.aad_length;
1300                 tag = op->sym->aead.digest.data;
1301                 if (tag == NULL)
1302                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1303                                 offset + srclen);
1304         }
1305
1306         taglen = sess->auth.digest_length;
1307
1308         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1309                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1310                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1311                         status = process_openssl_auth_encryption_gcm(
1312                                         mbuf_src, offset, srclen,
1313                                         aad, aadlen, iv,
1314                                         dst, tag, sess->cipher.ctx);
1315                 else
1316                         status = process_openssl_auth_encryption_ccm(
1317                                         mbuf_src, offset, srclen,
1318                                         aad, aadlen, iv,
1319                                         dst, tag, taglen, sess->cipher.ctx);
1320
1321         } else {
1322                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1323                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1324                         status = process_openssl_auth_decryption_gcm(
1325                                         mbuf_src, offset, srclen,
1326                                         aad, aadlen, iv,
1327                                         dst, tag, sess->cipher.ctx);
1328                 else
1329                         status = process_openssl_auth_decryption_ccm(
1330                                         mbuf_src, offset, srclen,
1331                                         aad, aadlen, iv,
1332                                         dst, tag, taglen, sess->cipher.ctx);
1333         }
1334
1335         if (status != 0) {
1336                 if (status == (-EFAULT) &&
1337                                 sess->auth.operation ==
1338                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1339                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1340                 else
1341                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1342         }
1343 }
1344
1345 /** Process cipher operation */
1346 static void
1347 process_openssl_cipher_op
1348                 (struct rte_crypto_op *op, struct openssl_session *sess,
1349                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1350 {
1351         uint8_t *dst, *iv;
1352         int srclen, status;
1353
1354         /*
1355          * Segmented destination buffer is not supported for
1356          * encryption/decryption
1357          */
1358         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1359                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1360                 return;
1361         }
1362
1363         srclen = op->sym->cipher.data.length;
1364         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1365                         op->sym->cipher.data.offset);
1366
1367         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1368                         sess->iv.offset);
1369
1370         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1371                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1372                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1373                                         op->sym->cipher.data.offset, iv,
1374                                         srclen, sess->cipher.ctx);
1375                 else
1376                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1377                                         op->sym->cipher.data.offset, iv,
1378                                         srclen, sess->cipher.ctx);
1379         else
1380                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1381                                 op->sym->cipher.data.offset, iv,
1382                                 sess->cipher.key.data, srclen,
1383                                 sess->cipher.ctx);
1384
1385         if (status != 0)
1386                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1387 }
1388
1389 /** Process cipher operation */
1390 static void
1391 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1392                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1393                 struct rte_mbuf *mbuf_dst)
1394 {
1395         uint8_t *src, *dst, *iv;
1396         uint8_t block_size, last_block_len;
1397         int srclen, status = 0;
1398
1399         srclen = op->sym->cipher.data.length;
1400         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1401                         op->sym->cipher.data.offset);
1402         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1403                         op->sym->cipher.data.offset);
1404
1405         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1406                         sess->iv.offset);
1407
1408         block_size = DES_BLOCK_SIZE;
1409
1410         last_block_len = srclen % block_size;
1411         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1412                 /* Encrypt only with ECB mode XOR IV */
1413                 if (srclen < block_size) {
1414                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1415                                         iv, srclen,
1416                                         sess->cipher.bpi_ctx);
1417                 } else {
1418                         srclen -= last_block_len;
1419                         /* Encrypt with the block aligned stream with CBC mode */
1420                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1421                                         op->sym->cipher.data.offset, iv,
1422                                         srclen, sess->cipher.ctx);
1423                         if (last_block_len) {
1424                                 /* Point at last block */
1425                                 dst += srclen;
1426                                 /*
1427                                  * IV is the last encrypted block from
1428                                  * the previous operation
1429                                  */
1430                                 iv = dst - block_size;
1431                                 src += srclen;
1432                                 srclen = last_block_len;
1433                                 /* Encrypt the last frame with ECB mode */
1434                                 status |= process_openssl_cipher_bpi_encrypt(src,
1435                                                 dst, iv,
1436                                                 srclen, sess->cipher.bpi_ctx);
1437                         }
1438                 }
1439         } else {
1440                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1441                 if (srclen < block_size) {
1442                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1443                                         iv,
1444                                         srclen,
1445                                         sess->cipher.bpi_ctx);
1446                 } else {
1447                         if (last_block_len) {
1448                                 /* Point at last block */
1449                                 dst += srclen - last_block_len;
1450                                 src += srclen - last_block_len;
1451                                 /*
1452                                  * IV is the last full block
1453                                  */
1454                                 iv = src - block_size;
1455                                 /*
1456                                  * Decrypt the last frame with ECB mode
1457                                  * (encrypt, as it is the same operation)
1458                                  */
1459                                 status = process_openssl_cipher_bpi_encrypt(src,
1460                                                 dst, iv,
1461                                                 last_block_len, sess->cipher.bpi_ctx);
1462                                 /* Prepare parameters for CBC mode op */
1463                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1464                                                 sess->iv.offset);
1465                                 dst += last_block_len - srclen;
1466                                 srclen -= last_block_len;
1467                         }
1468
1469                         /* Decrypt with CBC mode */
1470                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1471                                         op->sym->cipher.data.offset, iv,
1472                                         srclen, sess->cipher.ctx);
1473                 }
1474         }
1475
1476         if (status != 0)
1477                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1478 }
1479
1480 /** Process auth operation */
1481 static void
1482 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1483                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1484                 struct rte_mbuf *mbuf_dst)
1485 {
1486         uint8_t *dst;
1487         int srclen, status;
1488
1489         srclen = op->sym->auth.data.length;
1490
1491         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1492                 dst = qp->temp_digest;
1493         else {
1494                 dst = op->sym->auth.digest.data;
1495                 if (dst == NULL)
1496                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1497                                         op->sym->auth.data.offset +
1498                                         op->sym->auth.data.length);
1499         }
1500
1501         switch (sess->auth.mode) {
1502         case OPENSSL_AUTH_AS_AUTH:
1503                 status = process_openssl_auth(mbuf_src, dst,
1504                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1505                                 sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1506                 break;
1507         case OPENSSL_AUTH_AS_HMAC:
1508                 status = process_openssl_auth_hmac(mbuf_src, dst,
1509                                 op->sym->auth.data.offset, srclen,
1510                                 sess->auth.hmac.ctx);
1511                 break;
1512         default:
1513                 status = -1;
1514                 break;
1515         }
1516
1517         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1518                 if (memcmp(dst, op->sym->auth.digest.data,
1519                                 sess->auth.digest_length) != 0) {
1520                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1521                 }
1522         }
1523
1524         if (status != 0)
1525                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1526 }
1527
1528 /** Process crypto operation for mbuf */
1529 static int
1530 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1531                 struct openssl_session *sess)
1532 {
1533         struct rte_mbuf *msrc, *mdst;
1534         int retval;
1535
1536         msrc = op->sym->m_src;
1537         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1538
1539         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1540
1541         switch (sess->chain_order) {
1542         case OPENSSL_CHAIN_ONLY_CIPHER:
1543                 process_openssl_cipher_op(op, sess, msrc, mdst);
1544                 break;
1545         case OPENSSL_CHAIN_ONLY_AUTH:
1546                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1547                 break;
1548         case OPENSSL_CHAIN_CIPHER_AUTH:
1549                 process_openssl_cipher_op(op, sess, msrc, mdst);
1550                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
1551                 break;
1552         case OPENSSL_CHAIN_AUTH_CIPHER:
1553                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1554                 process_openssl_cipher_op(op, sess, msrc, mdst);
1555                 break;
1556         case OPENSSL_CHAIN_COMBINED:
1557                 process_openssl_combined_op(op, sess, msrc, mdst);
1558                 break;
1559         case OPENSSL_CHAIN_CIPHER_BPI:
1560                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1561                 break;
1562         default:
1563                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1564                 break;
1565         }
1566
1567         /* Free session if a session-less crypto op */
1568         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1569                 openssl_reset_session(sess);
1570                 memset(sess, 0, sizeof(struct openssl_session));
1571                 memset(op->sym->session, 0,
1572                                 rte_cryptodev_get_header_session_size());
1573                 rte_mempool_put(qp->sess_mp, sess);
1574                 rte_mempool_put(qp->sess_mp, op->sym->session);
1575                 op->sym->session = NULL;
1576         }
1577
1578         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1579                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1580
1581         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1582                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1583         else
1584                 retval = -1;
1585
1586         return retval;
1587 }
1588
1589 /*
1590  *------------------------------------------------------------------------------
1591  * PMD Framework
1592  *------------------------------------------------------------------------------
1593  */
1594
1595 /** Enqueue burst */
1596 static uint16_t
1597 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1598                 uint16_t nb_ops)
1599 {
1600         struct openssl_session *sess;
1601         struct openssl_qp *qp = queue_pair;
1602         int i, retval;
1603
1604         for (i = 0; i < nb_ops; i++) {
1605                 sess = get_session(qp, ops[i]);
1606                 if (unlikely(sess == NULL))
1607                         goto enqueue_err;
1608
1609                 retval = process_op(qp, ops[i], sess);
1610                 if (unlikely(retval < 0))
1611                         goto enqueue_err;
1612         }
1613
1614         qp->stats.enqueued_count += i;
1615         return i;
1616
1617 enqueue_err:
1618         qp->stats.enqueue_err_count++;
1619         return i;
1620 }
1621
1622 /** Dequeue burst */
1623 static uint16_t
1624 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1625                 uint16_t nb_ops)
1626 {
1627         struct openssl_qp *qp = queue_pair;
1628
1629         unsigned int nb_dequeued = 0;
1630
1631         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1632                         (void **)ops, nb_ops, NULL);
1633         qp->stats.dequeued_count += nb_dequeued;
1634
1635         return nb_dequeued;
1636 }
1637
1638 /** Create OPENSSL crypto device */
1639 static int
1640 cryptodev_openssl_create(const char *name,
1641                         struct rte_vdev_device *vdev,
1642                         struct rte_cryptodev_pmd_init_params *init_params)
1643 {
1644         struct rte_cryptodev *dev;
1645         struct openssl_private *internals;
1646
1647         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1648         if (dev == NULL) {
1649                 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1650                 goto init_error;
1651         }
1652
1653         dev->driver_id = cryptodev_driver_id;
1654         dev->dev_ops = rte_openssl_pmd_ops;
1655
1656         /* register rx/tx burst functions for data path */
1657         dev->dequeue_burst = openssl_pmd_dequeue_burst;
1658         dev->enqueue_burst = openssl_pmd_enqueue_burst;
1659
1660         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1661                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1662                         RTE_CRYPTODEV_FF_CPU_AESNI |
1663                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1664
1665         /* Set vector instructions mode supported */
1666         internals = dev->data->dev_private;
1667
1668         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1669         internals->max_nb_sessions = init_params->max_nb_sessions;
1670
1671         return 0;
1672
1673 init_error:
1674         OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1675                         init_params->name);
1676
1677         cryptodev_openssl_remove(vdev);
1678         return -EFAULT;
1679 }
1680
1681 /** Initialise OPENSSL crypto device */
1682 static int
1683 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1684 {
1685         struct rte_cryptodev_pmd_init_params init_params = {
1686                 "",
1687                 sizeof(struct openssl_private),
1688                 rte_socket_id(),
1689                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
1690                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
1691         };
1692         const char *name;
1693         const char *input_args;
1694
1695         name = rte_vdev_device_name(vdev);
1696         if (name == NULL)
1697                 return -EINVAL;
1698         input_args = rte_vdev_device_args(vdev);
1699
1700         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
1701
1702         return cryptodev_openssl_create(name, vdev, &init_params);
1703 }
1704
1705 /** Uninitialise OPENSSL crypto device */
1706 static int
1707 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1708 {
1709         struct rte_cryptodev *cryptodev;
1710         const char *name;
1711
1712         name = rte_vdev_device_name(vdev);
1713         if (name == NULL)
1714                 return -EINVAL;
1715
1716         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1717         if (cryptodev == NULL)
1718                 return -ENODEV;
1719
1720         return rte_cryptodev_pmd_destroy(cryptodev);
1721 }
1722
1723 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1724         .probe = cryptodev_openssl_probe,
1725         .remove = cryptodev_openssl_remove
1726 };
1727
1728 static struct cryptodev_driver openssl_crypto_drv;
1729
1730 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1731         cryptodev_openssl_pmd_drv);
1732 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1733         "max_nb_queue_pairs=<int> "
1734         "max_nb_sessions=<int> "
1735         "socket_id=<int>");
1736 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
1737                 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);