drivers/crypto/openssl/rte_openssl_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_common.h>
  34 #include <rte_hexdump.h>
  35 #include <rte_cryptodev.h>
  36 #include <rte_cryptodev_pmd.h>
  37 #include <rte_cryptodev_vdev.h>
  38 #include <rte_vdev.h>
  39 #include <rte_malloc.h>
  40 #include <rte_cpuflags.h>
  41
  42 #include <openssl/hmac.h>
  43 #include <openssl/evp.h>
  44
  45 #include "rte_openssl_pmd_private.h"
  46
  47 #define DES_BLOCK_SIZE 8
  48
  49 static uint8_t cryptodev_driver_id;
  50
  51 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  52 static HMAC_CTX *HMAC_CTX_new(void)
  53 {
  54         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  55
  56         if (ctx != NULL)
  57                 HMAC_CTX_init(ctx);
  58         return ctx;
  59 }
  60
  61 static void HMAC_CTX_free(HMAC_CTX *ctx)
  62 {
  63         if (ctx != NULL) {
  64                 HMAC_CTX_cleanup(ctx);
  65                 OPENSSL_free(ctx);
  66         }
  67 }
  68 #endif
  69
  70 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  71
  72 /*----------------------------------------------------------------------------*/
  73
  74 /**
  75  * Increment counter by 1
  76  * Counter is 64 bit array, big-endian
  77  */
  78 static void
  79 ctr_inc(uint8_t *ctr)
  80 {
  81         uint64_t *ctr64 = (uint64_t *)ctr;
  82
  83         *ctr64 = __builtin_bswap64(*ctr64);
  84         (*ctr64)++;
  85         *ctr64 = __builtin_bswap64(*ctr64);
  86 }
  87
  88 /*
  89  *------------------------------------------------------------------------------
  90  * Session Prepare
  91  *------------------------------------------------------------------------------
  92  */
  93
  94 /** Get xform chain order */
  95 static enum openssl_chain_order
  96 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
  97 {
  98         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
  99
 100         if (xform != NULL) {
 101                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 102                         if (xform->next == NULL)
 103                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
 104                         else if (xform->next->type ==
 105                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
 106                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
 107                 }
 108                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 109                         if (xform->next == NULL)
 110                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
 111                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 112                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
 113                 }
 114                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 115                         res = OPENSSL_CHAIN_COMBINED;
 116         }
 117
 118         return res;
 119 }
 120
 121 /** Get session cipher key from input cipher key */
 122 static void
 123 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
 124 {
 125         memcpy(session_key, input_key, keylen);
 126 }
 127
 128 /** Get key ede 24 bytes standard from input key */
 129 static int
 130 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
 131 {
 132         int res = 0;
 133
 134         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 135         switch (keylen) {
 136         case 24:
 137                 memcpy(key_ede, key, 24);
 138                 break;
 139         case 16:
 140                 /* K3 = K1 */
 141                 memcpy(key_ede, key, 16);
 142                 memcpy(key_ede + 16, key, 8);
 143                 break;
 144         case 8:
 145                 /* K1 = K2 = K3 (DES compatibility) */
 146                 memcpy(key_ede, key, 8);
 147                 memcpy(key_ede + 8, key, 8);
 148                 memcpy(key_ede + 16, key, 8);
 149                 break;
 150         default:
 151                 OPENSSL_LOG_ERR("Unsupported key size");
 152                 res = -EINVAL;
 153         }
 154
 155         return res;
 156 }
 157
 158 /** Get adequate openssl function for input cipher algorithm */
 159 static uint8_t
 160 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 161                 const EVP_CIPHER **algo)
 162 {
 163         int res = 0;
 164
 165         if (algo != NULL) {
 166                 switch (sess_algo) {
 167                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 168                         switch (keylen) {
 169                         case 16:
 170                                 *algo = EVP_des_ede_cbc();
 171                                 break;
 172                         case 24:
 173                                 *algo = EVP_des_ede3_cbc();
 174                                 break;
 175                         default:
 176                                 res = -EINVAL;
 177                         }
 178                         break;
 179                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 180                         break;
 181                 case RTE_CRYPTO_CIPHER_AES_CBC:
 182                         switch (keylen) {
 183                         case 16:
 184                                 *algo = EVP_aes_128_cbc();
 185                                 break;
 186                         case 24:
 187                                 *algo = EVP_aes_192_cbc();
 188                                 break;
 189                         case 32:
 190                                 *algo = EVP_aes_256_cbc();
 191                                 break;
 192                         default:
 193                                 res = -EINVAL;
 194                         }
 195                         break;
 196                 case RTE_CRYPTO_CIPHER_AES_CTR:
 197                         switch (keylen) {
 198                         case 16:
 199                                 *algo = EVP_aes_128_ctr();
 200                                 break;
 201                         case 24:
 202                                 *algo = EVP_aes_192_ctr();
 203                                 break;
 204                         case 32:
 205                                 *algo = EVP_aes_256_ctr();
 206                                 break;
 207                         default:
 208                                 res = -EINVAL;
 209                         }
 210                         break;
 211                 default:
 212                         res = -EINVAL;
 213                         break;
 214                 }
 215         } else {
 216                 res = -EINVAL;
 217         }
 218
 219         return res;
 220 }
 221
 222 /** Get adequate openssl function for input auth algorithm */
 223 static uint8_t
 224 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 225                 const EVP_MD **algo)
 226 {
 227         int res = 0;
 228
 229         if (algo != NULL) {
 230                 switch (sessalgo) {
 231                 case RTE_CRYPTO_AUTH_MD5:
 232                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 233                         *algo = EVP_md5();
 234                         break;
 235                 case RTE_CRYPTO_AUTH_SHA1:
 236                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 237                         *algo = EVP_sha1();
 238                         break;
 239                 case RTE_CRYPTO_AUTH_SHA224:
 240                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 241                         *algo = EVP_sha224();
 242                         break;
 243                 case RTE_CRYPTO_AUTH_SHA256:
 244                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 245                         *algo = EVP_sha256();
 246                         break;
 247                 case RTE_CRYPTO_AUTH_SHA384:
 248                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 249                         *algo = EVP_sha384();
 250                         break;
 251                 case RTE_CRYPTO_AUTH_SHA512:
 252                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 253                         *algo = EVP_sha512();
 254                         break;
 255                 default:
 256                         res = -EINVAL;
 257                         break;
 258                 }
 259         } else {
 260                 res = -EINVAL;
 261         }
 262
 263         return res;
 264 }
 265
 266 /** Get adequate openssl function for input cipher algorithm */
 267 static uint8_t
 268 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 269                 const EVP_CIPHER **algo)
 270 {
 271         int res = 0;
 272
 273         if (algo != NULL) {
 274                 switch (sess_algo) {
 275                 case RTE_CRYPTO_AEAD_AES_GCM:
 276                         switch (keylen) {
 277                         case 16:
 278                                 *algo = EVP_aes_128_gcm();
 279                                 break;
 280                         case 24:
 281                                 *algo = EVP_aes_192_gcm();
 282                                 break;
 283                         case 32:
 284                                 *algo = EVP_aes_256_gcm();
 285                                 break;
 286                         default:
 287                                 res = -EINVAL;
 288                         }
 289                         break;
 290                 case RTE_CRYPTO_AEAD_AES_CCM:
 291                         switch (keylen) {
 292                         case 16:
 293                                 *algo = EVP_aes_128_ccm();
 294                                 break;
 295                         case 24:
 296                                 *algo = EVP_aes_192_ccm();
 297                                 break;
 298                         case 32:
 299                                 *algo = EVP_aes_256_ccm();
 300                                 break;
 301                         default:
 302                                 res = -EINVAL;
 303                         }
 304                         break;
 305                 default:
 306                         res = -EINVAL;
 307                         break;
 308                 }
 309         } else {
 310                 res = -EINVAL;
 311         }
 312
 313         return res;
 314 }
 315
 316 /* Set session AEAD encryption parameters */
 317 static int
 318 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
 319                 enum rte_crypto_aead_algorithm algo,
 320                 uint8_t tag_len, uint8_t *key)
 321 {
 322         int iv_type = 0;
 323         unsigned int do_ccm;
 324
 325         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 326         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 327
 328         /* Select AEAD algo */
 329         switch (algo) {
 330         case RTE_CRYPTO_AEAD_AES_GCM:
 331                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 332                 if (tag_len != 16)
 333                         return -EINVAL;
 334                 do_ccm = 0;
 335                 break;
 336         case RTE_CRYPTO_AEAD_AES_CCM:
 337                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 338                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 339                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 340                         return -EINVAL;
 341                 do_ccm = 1;
 342                 break;
 343         default:
 344                 return -ENOTSUP;
 345         }
 346
 347         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 348         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 349
 350         if (get_aead_algo(algo, sess->cipher.key.length,
 351                         &sess->cipher.evp_algo) != 0)
 352                 return -EINVAL;
 353
 354         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 355
 356         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 357
 358         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 359                         NULL, NULL, NULL) <= 0)
 360                 return -EINVAL;
 361
 362         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
 363                         NULL) <= 0)
 364                 return -EINVAL;
 365
 366         if (do_ccm)
 367                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 368                                 tag_len, NULL);
 369
 370         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 371                 return -EINVAL;
 372
 373         return 0;
 374 }
 375
 376 /* Set session AEAD decryption parameters */
 377 static int
 378 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
 379                 enum rte_crypto_aead_algorithm algo,
 380                 uint8_t tag_len, uint8_t *key)
 381 {
 382         int iv_type = 0;
 383         unsigned int do_ccm = 0;
 384
 385         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 386         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 387
 388         /* Select AEAD algo */
 389         switch (algo) {
 390         case RTE_CRYPTO_AEAD_AES_GCM:
 391                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 392                 if (tag_len != 16)
 393                         return -EINVAL;
 394                 break;
 395         case RTE_CRYPTO_AEAD_AES_CCM:
 396                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 397                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 398                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 399                         return -EINVAL;
 400                 do_ccm = 1;
 401                 break;
 402         default:
 403                 return -ENOTSUP;
 404         }
 405
 406         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 407         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 408
 409         if (get_aead_algo(algo, sess->cipher.key.length,
 410                         &sess->cipher.evp_algo) != 0)
 411                 return -EINVAL;
 412
 413         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 414
 415         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 416
 417         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 418                         NULL, NULL, NULL) <= 0)
 419                 return -EINVAL;
 420
 421         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
 422                         sess->iv.length, NULL) <= 0)
 423                 return -EINVAL;
 424
 425         if (do_ccm)
 426                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 427                                 tag_len, NULL);
 428
 429         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 430                 return -EINVAL;
 431
 432         return 0;
 433 }
 434
 435 /** Set session cipher parameters */
 436 static int
 437 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 438                 const struct rte_crypto_sym_xform *xform)
 439 {
 440         /* Select cipher direction */
 441         sess->cipher.direction = xform->cipher.op;
 442         /* Select cipher key */
 443         sess->cipher.key.length = xform->cipher.key.length;
 444
 445         /* Set IV parameters */
 446         sess->iv.offset = xform->cipher.iv.offset;
 447         sess->iv.length = xform->cipher.iv.length;
 448
 449         /* Select cipher algo */
 450         switch (xform->cipher.algo) {
 451         case RTE_CRYPTO_CIPHER_3DES_CBC:
 452         case RTE_CRYPTO_CIPHER_AES_CBC:
 453         case RTE_CRYPTO_CIPHER_AES_CTR:
 454                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 455                 sess->cipher.algo = xform->cipher.algo;
 456                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 457
 458                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 459                                 &sess->cipher.evp_algo) != 0)
 460                         return -EINVAL;
 461
 462                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 463                         sess->cipher.key.data);
 464                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 465                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 466                                         sess->cipher.evp_algo,
 467                                         NULL, xform->cipher.key.data,
 468                                         NULL) != 1) {
 469                                 return -EINVAL;
 470                         }
 471                 } else if (sess->cipher.direction ==
 472                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 473                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 474                                         sess->cipher.evp_algo,
 475                                         NULL, xform->cipher.key.data,
 476                                         NULL) != 1) {
 477                                 return -EINVAL;
 478                         }
 479                 }
 480
 481                 break;
 482
 483         case RTE_CRYPTO_CIPHER_3DES_CTR:
 484                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 485                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 486
 487                 if (get_cipher_key_ede(xform->cipher.key.data,
 488                                 sess->cipher.key.length,
 489                                 sess->cipher.key.data) != 0)
 490                         return -EINVAL;
 491                 break;
 492
 493         case RTE_CRYPTO_CIPHER_DES_CBC:
 494                 sess->cipher.algo = xform->cipher.algo;
 495                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 496                 sess->cipher.evp_algo = EVP_des_cbc();
 497
 498                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 499                         sess->cipher.key.data);
 500                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 501                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 502                                         sess->cipher.evp_algo,
 503                                         NULL, xform->cipher.key.data,
 504                                         NULL) != 1) {
 505                                 return -EINVAL;
 506                         }
 507                 } else if (sess->cipher.direction ==
 508                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 509                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 510                                         sess->cipher.evp_algo,
 511                                         NULL, xform->cipher.key.data,
 512                                         NULL) != 1) {
 513                                 return -EINVAL;
 514                         }
 515                 }
 516
 517                 break;
 518
 519         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 520                 sess->cipher.algo = xform->cipher.algo;
 521                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 522                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 523                 sess->cipher.evp_algo = EVP_des_cbc();
 524
 525                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 526                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 527                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 528                                 NULL, xform->cipher.key.data, 0) != 1)
 529                         return -EINVAL;
 530
 531                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 532                         sess->cipher.key.data);
 533                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 534                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 535                                         sess->cipher.evp_algo,
 536                                         NULL, xform->cipher.key.data,
 537                                         NULL) != 1) {
 538                                 return -EINVAL;
 539                         }
 540                 } else if (sess->cipher.direction ==
 541                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 542                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 543                                         sess->cipher.evp_algo,
 544                                         NULL, xform->cipher.key.data,
 545                                         NULL) != 1) {
 546                                 return -EINVAL;
 547                         }
 548                 }
 549
 550                 break;
 551         default:
 552                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 553                 return -ENOTSUP;
 554         }
 555
 556         return 0;
 557 }
 558
 559 /* Set session auth parameters */
 560 static int
 561 openssl_set_session_auth_parameters(struct openssl_session *sess,
 562                 const struct rte_crypto_sym_xform *xform)
 563 {
 564         /* Select auth generate/verify */
 565         sess->auth.operation = xform->auth.op;
 566         sess->auth.algo = xform->auth.algo;
 567
 568         sess->auth.digest_length = xform->auth.digest_length;
 569
 570         /* Select auth algo */
 571         switch (xform->auth.algo) {
 572         case RTE_CRYPTO_AUTH_AES_GMAC:
 573                 /*
 574                  * OpenSSL requires GMAC to be a GCM operation
 575                  * with no cipher data length
 576                  */
 577                 sess->cipher.key.length = xform->auth.key.length;
 578
 579                 /* Set IV parameters */
 580                 sess->iv.offset = xform->auth.iv.offset;
 581                 sess->iv.length = xform->auth.iv.length;
 582
 583                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 584                         return openssl_set_sess_aead_enc_param(sess,
 585                                                 RTE_CRYPTO_AEAD_AES_GCM,
 586                                                 xform->auth.digest_length,
 587                                                 xform->auth.key.data);
 588                 else
 589                         return openssl_set_sess_aead_dec_param(sess,
 590                                                 RTE_CRYPTO_AEAD_AES_GCM,
 591                                                 xform->auth.digest_length,
 592                                                 xform->auth.key.data);
 593                 break;
 594
 595         case RTE_CRYPTO_AUTH_MD5:
 596         case RTE_CRYPTO_AUTH_SHA1:
 597         case RTE_CRYPTO_AUTH_SHA224:
 598         case RTE_CRYPTO_AUTH_SHA256:
 599         case RTE_CRYPTO_AUTH_SHA384:
 600         case RTE_CRYPTO_AUTH_SHA512:
 601                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 602                 if (get_auth_algo(xform->auth.algo,
 603                                 &sess->auth.auth.evp_algo) != 0)
 604                         return -EINVAL;
 605                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 606                 break;
 607
 608         case RTE_CRYPTO_AUTH_MD5_HMAC:
 609         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 610         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 611         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 612         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 613         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 614                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 615                 sess->auth.hmac.ctx = HMAC_CTX_new();
 616                 if (get_auth_algo(xform->auth.algo,
 617                                 &sess->auth.hmac.evp_algo) != 0)
 618                         return -EINVAL;
 619
 620                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 621                                 xform->auth.key.data,
 622                                 xform->auth.key.length,
 623                                 sess->auth.hmac.evp_algo, NULL) != 1)
 624                         return -EINVAL;
 625                 break;
 626
 627         default:
 628                 return -ENOTSUP;
 629         }
 630
 631         return 0;
 632 }
 633
 634 /* Set session AEAD parameters */
 635 static int
 636 openssl_set_session_aead_parameters(struct openssl_session *sess,
 637                 const struct rte_crypto_sym_xform *xform)
 638 {
 639         /* Select cipher key */
 640         sess->cipher.key.length = xform->aead.key.length;
 641
 642         /* Set IV parameters */
 643         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 644                 /*
 645                  * For AES-CCM, the actual IV is placed
 646                  * one byte after the start of the IV field,
 647                  * according to the API.
 648                  */
 649                 sess->iv.offset = xform->aead.iv.offset + 1;
 650         else
 651                 sess->iv.offset = xform->aead.iv.offset;
 652
 653         sess->iv.length = xform->aead.iv.length;
 654
 655         sess->auth.aad_length = xform->aead.aad_length;
 656         sess->auth.digest_length = xform->aead.digest_length;
 657
 658         sess->aead_algo = xform->aead.algo;
 659         /* Select cipher direction */
 660         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 661                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
 662                                 xform->aead.digest_length, xform->aead.key.data);
 663         else
 664                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
 665                                 xform->aead.digest_length, xform->aead.key.data);
 666 }
 667
 668 /** Parse crypto xform chain and set private session parameters */
 669 int
 670 openssl_set_session_parameters(struct openssl_session *sess,
 671                 const struct rte_crypto_sym_xform *xform)
 672 {
 673         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 674         const struct rte_crypto_sym_xform *auth_xform = NULL;
 675         const struct rte_crypto_sym_xform *aead_xform = NULL;
 676         int ret;
 677
 678         sess->chain_order = openssl_get_chain_order(xform);
 679         switch (sess->chain_order) {
 680         case OPENSSL_CHAIN_ONLY_CIPHER:
 681                 cipher_xform = xform;
 682                 break;
 683         case OPENSSL_CHAIN_ONLY_AUTH:
 684                 auth_xform = xform;
 685                 break;
 686         case OPENSSL_CHAIN_CIPHER_AUTH:
 687                 cipher_xform = xform;
 688                 auth_xform = xform->next;
 689                 break;
 690         case OPENSSL_CHAIN_AUTH_CIPHER:
 691                 auth_xform = xform;
 692                 cipher_xform = xform->next;
 693                 break;
 694         case OPENSSL_CHAIN_COMBINED:
 695                 aead_xform = xform;
 696                 break;
 697         default:
 698                 return -EINVAL;
 699         }
 700
 701         /* Default IV length = 0 */
 702         sess->iv.length = 0;
 703
 704         /* cipher_xform must be check before auth_xform */
 705         if (cipher_xform) {
 706                 ret = openssl_set_session_cipher_parameters(
 707                                 sess, cipher_xform);
 708                 if (ret != 0) {
 709                         OPENSSL_LOG_ERR(
 710                                 "Invalid/unsupported cipher parameters");
 711                         return ret;
 712                 }
 713         }
 714
 715         if (auth_xform) {
 716                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 717                 if (ret != 0) {
 718                         OPENSSL_LOG_ERR(
 719                                 "Invalid/unsupported auth parameters");
 720                         return ret;
 721                 }
 722         }
 723
 724         if (aead_xform) {
 725                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 726                 if (ret != 0) {
 727                         OPENSSL_LOG_ERR(
 728                                 "Invalid/unsupported AEAD parameters");
 729                         return ret;
 730                 }
 731         }
 732
 733         return 0;
 734 }
 735
 736 /** Reset private session parameters */
 737 void
 738 openssl_reset_session(struct openssl_session *sess)
 739 {
 740         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 741
 742         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 743                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 744
 745         switch (sess->auth.mode) {
 746         case OPENSSL_AUTH_AS_AUTH:
 747                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 748                 break;
 749         case OPENSSL_AUTH_AS_HMAC:
 750                 EVP_PKEY_free(sess->auth.hmac.pkey);
 751                 HMAC_CTX_free(sess->auth.hmac.ctx);
 752                 break;
 753         default:
 754                 break;
 755         }
 756 }
 757
 758 /** Provide session for operation */
 759 static struct openssl_session *
 760 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 761 {
 762         struct openssl_session *sess = NULL;
 763
 764         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 765                 /* get existing session */
 766                 if (likely(op->sym->session != NULL))
 767                         sess = (struct openssl_session *)
 768                                         get_session_private_data(
 769                                         op->sym->session,
 770                                         cryptodev_driver_id);
 771         } else {
 772                 /* provide internal session */
 773                 void *_sess = NULL;
 774                 void *_sess_private_data = NULL;
 775
 776                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 777                         return NULL;
 778
 779                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 780                         return NULL;
 781
 782                 sess = (struct openssl_session *)_sess_private_data;
 783
 784                 if (unlikely(openssl_set_session_parameters(sess,
 785                                 op->sym->xform) != 0)) {
 786                         rte_mempool_put(qp->sess_mp, _sess);
 787                         rte_mempool_put(qp->sess_mp, _sess_private_data);
 788                         sess = NULL;
 789                 }
 790                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 791                 set_session_private_data(op->sym->session, cryptodev_driver_id,
 792                         _sess_private_data);
 793         }
 794
 795         if (sess == NULL)
 796                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 797
 798         return sess;
 799 }
 800
 801 /*
 802  *------------------------------------------------------------------------------
 803  * Process Operations
 804  *------------------------------------------------------------------------------
 805  */
 806 static inline int
 807 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 808                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 809 {
 810         struct rte_mbuf *m;
 811         int dstlen;
 812         int l, n = srclen;
 813         uint8_t *src;
 814
 815         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 816                         m = m->next)
 817                 offset -= rte_pktmbuf_data_len(m);
 818
 819         if (m == 0)
 820                 return -1;
 821
 822         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 823
 824         l = rte_pktmbuf_data_len(m) - offset;
 825         if (srclen <= l) {
 826                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 827                         return -1;
 828                 *dst += l;
 829                 return 0;
 830         }
 831
 832         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 833                 return -1;
 834
 835         *dst += dstlen;
 836         n -= l;
 837
 838         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 839                 src = rte_pktmbuf_mtod(m, uint8_t *);
 840                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 841                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 842                         return -1;
 843                 *dst += dstlen;
 844                 n -= l;
 845         }
 846
 847         return 0;
 848 }
 849
 850 static inline int
 851 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 852                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 853 {
 854         struct rte_mbuf *m;
 855         int dstlen;
 856         int l, n = srclen;
 857         uint8_t *src;
 858
 859         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 860                         m = m->next)
 861                 offset -= rte_pktmbuf_data_len(m);
 862
 863         if (m == 0)
 864                 return -1;
 865
 866         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 867
 868         l = rte_pktmbuf_data_len(m) - offset;
 869         if (srclen <= l) {
 870                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 871                         return -1;
 872                 *dst += l;
 873                 return 0;
 874         }
 875
 876         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 877                 return -1;
 878
 879         *dst += dstlen;
 880         n -= l;
 881
 882         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 883                 src = rte_pktmbuf_mtod(m, uint8_t *);
 884                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 885                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 886                         return -1;
 887                 *dst += dstlen;
 888                 n -= l;
 889         }
 890
 891         return 0;
 892 }
 893
 894 /** Process standard openssl cipher encryption */
 895 static int
 896 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 897                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 898 {
 899         int totlen;
 900
 901         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 902                 goto process_cipher_encrypt_err;
 903
 904         EVP_CIPHER_CTX_set_padding(ctx, 0);
 905
 906         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 907                         srclen, ctx))
 908                 goto process_cipher_encrypt_err;
 909
 910         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
 911                 goto process_cipher_encrypt_err;
 912
 913         return 0;
 914
 915 process_cipher_encrypt_err:
 916         OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
 917         return -EINVAL;
 918 }
 919
 920 /** Process standard openssl cipher encryption */
 921 static int
 922 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 923                 uint8_t *iv, int srclen,
 924                 EVP_CIPHER_CTX *ctx)
 925 {
 926         uint8_t i;
 927         uint8_t encrypted_iv[DES_BLOCK_SIZE];
 928         int encrypted_ivlen;
 929
 930         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
 931                         iv, DES_BLOCK_SIZE) <= 0)
 932                 goto process_cipher_encrypt_err;
 933
 934         for (i = 0; i < srclen; i++)
 935                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
 936
 937         return 0;
 938
 939 process_cipher_encrypt_err:
 940         OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
 941         return -EINVAL;
 942 }
 943 /** Process standard openssl cipher decryption */
 944 static int
 945 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 946                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 947 {
 948         int totlen;
 949
 950         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 951                 goto process_cipher_decrypt_err;
 952
 953         EVP_CIPHER_CTX_set_padding(ctx, 0);
 954
 955         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 956                         srclen, ctx))
 957                 goto process_cipher_decrypt_err;
 958
 959         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
 960                 goto process_cipher_decrypt_err;
 961         return 0;
 962
 963 process_cipher_decrypt_err:
 964         OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
 965         return -EINVAL;
 966 }
 967
 968 /** Process cipher des 3 ctr encryption, decryption algorithm */
 969 static int
 970 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
 971                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 972                 EVP_CIPHER_CTX *ctx)
 973 {
 974         uint8_t ebuf[8], ctr[8];
 975         int unused, n;
 976         struct rte_mbuf *m;
 977         uint8_t *src;
 978         int l;
 979
 980         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 981                         m = m->next)
 982                 offset -= rte_pktmbuf_data_len(m);
 983
 984         if (m == 0)
 985                 goto process_cipher_des3ctr_err;
 986
 987         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 988         l = rte_pktmbuf_data_len(m) - offset;
 989
 990         /* We use 3DES encryption also for decryption.
 991          * IV is not important for 3DES ecb
 992          */
 993         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
 994                 goto process_cipher_des3ctr_err;
 995
 996         memcpy(ctr, iv, 8);
 997
 998         for (n = 0; n < srclen; n++) {
 999                 if (n % 8 == 0) {
1000                         if (EVP_EncryptUpdate(ctx,
1001                                         (unsigned char *)&ebuf, &unused,
1002                                         (const unsigned char *)&ctr, 8) <= 0)
1003                                 goto process_cipher_des3ctr_err;
1004                         ctr_inc(ctr);
1005                 }
1006                 dst[n] = *(src++) ^ ebuf[n % 8];
1007
1008                 l--;
1009                 if (!l) {
1010                         m = m->next;
1011                         if (m) {
1012                                 src = rte_pktmbuf_mtod(m, uint8_t *);
1013                                 l = rte_pktmbuf_data_len(m);
1014                         }
1015                 }
1016         }
1017
1018         return 0;
1019
1020 process_cipher_des3ctr_err:
1021         OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
1022         return -EINVAL;
1023 }
1024
1025 /** Process AES-GCM encrypt algorithm */
1026 static int
1027 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1028                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1029                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1030 {
1031         int len = 0, unused = 0;
1032         uint8_t empty[] = {};
1033
1034         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1035                 goto process_auth_encryption_gcm_err;
1036
1037         if (aadlen > 0)
1038                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1039                         goto process_auth_encryption_gcm_err;
1040
1041         if (srclen > 0)
1042                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1043                                 srclen, ctx))
1044                         goto process_auth_encryption_gcm_err;
1045
1046         /* Workaround open ssl bug in version less then 1.0.1f */
1047         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1048                 goto process_auth_encryption_gcm_err;
1049
1050         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1051                 goto process_auth_encryption_gcm_err;
1052
1053         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1054                 goto process_auth_encryption_gcm_err;
1055
1056         return 0;
1057
1058 process_auth_encryption_gcm_err:
1059         OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
1060         return -EINVAL;
1061 }
1062
1063 /** Process AES-CCM encrypt algorithm */
1064 static int
1065 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1066                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1067                 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1068 {
1069         int len = 0;
1070
1071         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1072                 goto process_auth_encryption_ccm_err;
1073
1074         if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1075                 goto process_auth_encryption_ccm_err;
1076
1077         if (aadlen > 0)
1078                 /*
1079                  * For AES-CCM, the actual AAD is placed
1080                  * 18 bytes after the start of the AAD field,
1081                  * according to the API.
1082                  */
1083                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1084                         goto process_auth_encryption_ccm_err;
1085
1086         if (srclen > 0)
1087                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1088                                 srclen, ctx))
1089                         goto process_auth_encryption_ccm_err;
1090
1091         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1092                 goto process_auth_encryption_ccm_err;
1093
1094         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1095                 goto process_auth_encryption_ccm_err;
1096
1097         return 0;
1098
1099 process_auth_encryption_ccm_err:
1100         OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed");
1101         return -EINVAL;
1102 }
1103
1104 /** Process AES-GCM decrypt algorithm */
1105 static int
1106 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1107                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1108                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1109 {
1110         int len = 0, unused = 0;
1111         uint8_t empty[] = {};
1112
1113         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1114                 goto process_auth_decryption_gcm_err;
1115
1116         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1117                 goto process_auth_decryption_gcm_err;
1118
1119         if (aadlen > 0)
1120                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1121                         goto process_auth_decryption_gcm_err;
1122
1123         if (srclen > 0)
1124                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1125                                 srclen, ctx))
1126                         goto process_auth_decryption_gcm_err;
1127
1128         /* Workaround open ssl bug in version less then 1.0.1f */
1129         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1130                 goto process_auth_decryption_gcm_err;
1131
1132         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1133                 return -EFAULT;
1134
1135         return 0;
1136
1137 process_auth_decryption_gcm_err:
1138         OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed");
1139         return -EINVAL;
1140 }
1141
1142 /** Process AES-CCM decrypt algorithm */
1143 static int
1144 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1145                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1146                 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1147                 EVP_CIPHER_CTX *ctx)
1148 {
1149         int len = 0;
1150
1151         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1152                 goto process_auth_decryption_ccm_err;
1153
1154         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1155                 goto process_auth_decryption_ccm_err;
1156
1157         if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1158                 goto process_auth_decryption_ccm_err;
1159
1160         if (aadlen > 0)
1161                 /*
1162                  * For AES-CCM, the actual AAD is placed
1163                  * 18 bytes after the start of the AAD field,
1164                  * according to the API.
1165                  */
1166                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1167                         goto process_auth_decryption_ccm_err;
1168
1169         if (srclen > 0)
1170                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1171                                 srclen, ctx))
1172                         return -EFAULT;
1173
1174         return 0;
1175
1176 process_auth_decryption_ccm_err:
1177         OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed");
1178         return -EINVAL;
1179 }
1180
1181 /** Process standard openssl auth algorithms */
1182 static int
1183 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1184                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1185                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1186 {
1187         size_t dstlen;
1188         struct rte_mbuf *m;
1189         int l, n = srclen;
1190         uint8_t *src;
1191
1192         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1193                         m = m->next)
1194                 offset -= rte_pktmbuf_data_len(m);
1195
1196         if (m == 0)
1197                 goto process_auth_err;
1198
1199         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1200                 goto process_auth_err;
1201
1202         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1203
1204         l = rte_pktmbuf_data_len(m) - offset;
1205         if (srclen <= l) {
1206                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1207                         goto process_auth_err;
1208                 goto process_auth_final;
1209         }
1210
1211         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1212                 goto process_auth_err;
1213
1214         n -= l;
1215
1216         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1217                 src = rte_pktmbuf_mtod(m, uint8_t *);
1218                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1219                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1220                         goto process_auth_err;
1221                 n -= l;
1222         }
1223
1224 process_auth_final:
1225         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1226                 goto process_auth_err;
1227         return 0;
1228
1229 process_auth_err:
1230         OPENSSL_LOG_ERR("Process openssl auth failed");
1231         return -EINVAL;
1232 }
1233
1234 /** Process standard openssl auth algorithms with hmac */
1235 static int
1236 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1237                 int srclen, HMAC_CTX *ctx)
1238 {
1239         unsigned int dstlen;
1240         struct rte_mbuf *m;
1241         int l, n = srclen;
1242         uint8_t *src;
1243
1244         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1245                         m = m->next)
1246                 offset -= rte_pktmbuf_data_len(m);
1247
1248         if (m == 0)
1249                 goto process_auth_err;
1250
1251         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1252
1253         l = rte_pktmbuf_data_len(m) - offset;
1254         if (srclen <= l) {
1255                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1256                         goto process_auth_err;
1257                 goto process_auth_final;
1258         }
1259
1260         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1261                 goto process_auth_err;
1262
1263         n -= l;
1264
1265         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1266                 src = rte_pktmbuf_mtod(m, uint8_t *);
1267                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1268                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1269                         goto process_auth_err;
1270                 n -= l;
1271         }
1272
1273 process_auth_final:
1274         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1275                 goto process_auth_err;
1276
1277         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1278                 goto process_auth_err;
1279
1280         return 0;
1281
1282 process_auth_err:
1283         OPENSSL_LOG_ERR("Process openssl auth failed");
1284         return -EINVAL;
1285 }
1286
1287 /*----------------------------------------------------------------------------*/
1288
1289 /** Process auth/cipher combined operation */
1290 static void
1291 process_openssl_combined_op
1292                 (struct rte_crypto_op *op, struct openssl_session *sess,
1293                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1294 {
1295         /* cipher */
1296         uint8_t *dst = NULL, *iv, *tag, *aad;
1297         int srclen, aadlen, status = -1;
1298         uint32_t offset;
1299         uint8_t taglen;
1300
1301         /*
1302          * Segmented destination buffer is not supported for
1303          * encryption/decryption
1304          */
1305         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1306                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1307                 return;
1308         }
1309
1310         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1311                         sess->iv.offset);
1312         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1313                 srclen = 0;
1314                 offset = op->sym->auth.data.offset;
1315                 aadlen = op->sym->auth.data.length;
1316                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1317                                 op->sym->auth.data.offset);
1318                 tag = op->sym->auth.digest.data;
1319                 if (tag == NULL)
1320                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1321                                 offset + aadlen);
1322         } else {
1323                 srclen = op->sym->aead.data.length;
1324                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1325                                 op->sym->aead.data.offset);
1326                 offset = op->sym->aead.data.offset;
1327                 aad = op->sym->aead.aad.data;
1328                 aadlen = sess->auth.aad_length;
1329                 tag = op->sym->aead.digest.data;
1330                 if (tag == NULL)
1331                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1332                                 offset + srclen);
1333         }
1334
1335         taglen = sess->auth.digest_length;
1336
1337         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1338                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1339                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1340                         status = process_openssl_auth_encryption_gcm(
1341                                         mbuf_src, offset, srclen,
1342                                         aad, aadlen, iv,
1343                                         dst, tag, sess->cipher.ctx);
1344                 else
1345                         status = process_openssl_auth_encryption_ccm(
1346                                         mbuf_src, offset, srclen,
1347                                         aad, aadlen, iv,
1348                                         dst, tag, taglen, sess->cipher.ctx);
1349
1350         } else {
1351                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1352                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1353                         status = process_openssl_auth_decryption_gcm(
1354                                         mbuf_src, offset, srclen,
1355                                         aad, aadlen, iv,
1356                                         dst, tag, sess->cipher.ctx);
1357                 else
1358                         status = process_openssl_auth_decryption_ccm(
1359                                         mbuf_src, offset, srclen,
1360                                         aad, aadlen, iv,
1361                                         dst, tag, taglen, sess->cipher.ctx);
1362         }
1363
1364         if (status != 0) {
1365                 if (status == (-EFAULT) &&
1366                                 sess->auth.operation ==
1367                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1368                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1369                 else
1370                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1371         }
1372 }
1373
1374 /** Process cipher operation */
1375 static void
1376 process_openssl_cipher_op
1377                 (struct rte_crypto_op *op, struct openssl_session *sess,
1378                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1379 {
1380         uint8_t *dst, *iv;
1381         int srclen, status;
1382
1383         /*
1384          * Segmented destination buffer is not supported for
1385          * encryption/decryption
1386          */
1387         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1388                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1389                 return;
1390         }
1391
1392         srclen = op->sym->cipher.data.length;
1393         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1394                         op->sym->cipher.data.offset);
1395
1396         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1397                         sess->iv.offset);
1398
1399         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1400                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1401                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1402                                         op->sym->cipher.data.offset, iv,
1403                                         srclen, sess->cipher.ctx);
1404                 else
1405                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1406                                         op->sym->cipher.data.offset, iv,
1407                                         srclen, sess->cipher.ctx);
1408         else
1409                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1410                                 op->sym->cipher.data.offset, iv,
1411                                 sess->cipher.key.data, srclen,
1412                                 sess->cipher.ctx);
1413
1414         if (status != 0)
1415                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1416 }
1417
1418 /** Process cipher operation */
1419 static void
1420 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1421                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1422                 struct rte_mbuf *mbuf_dst)
1423 {
1424         uint8_t *src, *dst, *iv;
1425         uint8_t block_size, last_block_len;
1426         int srclen, status = 0;
1427
1428         srclen = op->sym->cipher.data.length;
1429         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1430                         op->sym->cipher.data.offset);
1431         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1432                         op->sym->cipher.data.offset);
1433
1434         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1435                         sess->iv.offset);
1436
1437         block_size = DES_BLOCK_SIZE;
1438
1439         last_block_len = srclen % block_size;
1440         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1441                 /* Encrypt only with ECB mode XOR IV */
1442                 if (srclen < block_size) {
1443                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1444                                         iv, srclen,
1445                                         sess->cipher.bpi_ctx);
1446                 } else {
1447                         srclen -= last_block_len;
1448                         /* Encrypt with the block aligned stream with CBC mode */
1449                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1450                                         op->sym->cipher.data.offset, iv,
1451                                         srclen, sess->cipher.ctx);
1452                         if (last_block_len) {
1453                                 /* Point at last block */
1454                                 dst += srclen;
1455                                 /*
1456                                  * IV is the last encrypted block from
1457                                  * the previous operation
1458                                  */
1459                                 iv = dst - block_size;
1460                                 src += srclen;
1461                                 srclen = last_block_len;
1462                                 /* Encrypt the last frame with ECB mode */
1463                                 status |= process_openssl_cipher_bpi_encrypt(src,
1464                                                 dst, iv,
1465                                                 srclen, sess->cipher.bpi_ctx);
1466                         }
1467                 }
1468         } else {
1469                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1470                 if (srclen < block_size) {
1471                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1472                                         iv,
1473                                         srclen,
1474                                         sess->cipher.bpi_ctx);
1475                 } else {
1476                         if (last_block_len) {
1477                                 /* Point at last block */
1478                                 dst += srclen - last_block_len;
1479                                 src += srclen - last_block_len;
1480                                 /*
1481                                  * IV is the last full block
1482                                  */
1483                                 iv = src - block_size;
1484                                 /*
1485                                  * Decrypt the last frame with ECB mode
1486                                  * (encrypt, as it is the same operation)
1487                                  */
1488                                 status = process_openssl_cipher_bpi_encrypt(src,
1489                                                 dst, iv,
1490                                                 last_block_len, sess->cipher.bpi_ctx);
1491                                 /* Prepare parameters for CBC mode op */
1492                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1493                                                 sess->iv.offset);
1494                                 dst += last_block_len - srclen;
1495                                 srclen -= last_block_len;
1496                         }
1497
1498                         /* Decrypt with CBC mode */
1499                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1500                                         op->sym->cipher.data.offset, iv,
1501                                         srclen, sess->cipher.ctx);
1502                 }
1503         }
1504
1505         if (status != 0)
1506                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1507 }
1508
1509 /** Process auth operation */
1510 static void
1511 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1512                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1513                 struct rte_mbuf *mbuf_dst)
1514 {
1515         uint8_t *dst;
1516         int srclen, status;
1517
1518         srclen = op->sym->auth.data.length;
1519
1520         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1521                 dst = qp->temp_digest;
1522         else {
1523                 dst = op->sym->auth.digest.data;
1524                 if (dst == NULL)
1525                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1526                                         op->sym->auth.data.offset +
1527                                         op->sym->auth.data.length);
1528         }
1529
1530         switch (sess->auth.mode) {
1531         case OPENSSL_AUTH_AS_AUTH:
1532                 status = process_openssl_auth(mbuf_src, dst,
1533                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1534                                 sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1535                 break;
1536         case OPENSSL_AUTH_AS_HMAC:
1537                 status = process_openssl_auth_hmac(mbuf_src, dst,
1538                                 op->sym->auth.data.offset, srclen,
1539                                 sess->auth.hmac.ctx);
1540                 break;
1541         default:
1542                 status = -1;
1543                 break;
1544         }
1545
1546         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1547                 if (memcmp(dst, op->sym->auth.digest.data,
1548                                 sess->auth.digest_length) != 0) {
1549                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1550                 }
1551         }
1552
1553         if (status != 0)
1554                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1555 }
1556
1557 /** Process crypto operation for mbuf */
1558 static int
1559 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1560                 struct openssl_session *sess)
1561 {
1562         struct rte_mbuf *msrc, *mdst;
1563         int retval;
1564
1565         msrc = op->sym->m_src;
1566         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1567
1568         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1569
1570         switch (sess->chain_order) {
1571         case OPENSSL_CHAIN_ONLY_CIPHER:
1572                 process_openssl_cipher_op(op, sess, msrc, mdst);
1573                 break;
1574         case OPENSSL_CHAIN_ONLY_AUTH:
1575                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1576                 break;
1577         case OPENSSL_CHAIN_CIPHER_AUTH:
1578                 process_openssl_cipher_op(op, sess, msrc, mdst);
1579                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
1580                 break;
1581         case OPENSSL_CHAIN_AUTH_CIPHER:
1582                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1583                 process_openssl_cipher_op(op, sess, msrc, mdst);
1584                 break;
1585         case OPENSSL_CHAIN_COMBINED:
1586                 process_openssl_combined_op(op, sess, msrc, mdst);
1587                 break;
1588         case OPENSSL_CHAIN_CIPHER_BPI:
1589                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1590                 break;
1591         default:
1592                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1593                 break;
1594         }
1595
1596         /* Free session if a session-less crypto op */
1597         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1598                 openssl_reset_session(sess);
1599                 memset(sess, 0, sizeof(struct openssl_session));
1600                 memset(op->sym->session, 0,
1601                                 rte_cryptodev_get_header_session_size());
1602                 rte_mempool_put(qp->sess_mp, sess);
1603                 rte_mempool_put(qp->sess_mp, op->sym->session);
1604                 op->sym->session = NULL;
1605         }
1606
1607         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1608                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1609
1610         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1611                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1612         else
1613                 retval = -1;
1614
1615         return retval;
1616 }
1617
1618 /*
1619  *------------------------------------------------------------------------------
1620  * PMD Framework
1621  *------------------------------------------------------------------------------
1622  */
1623
1624 /** Enqueue burst */
1625 static uint16_t
1626 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1627                 uint16_t nb_ops)
1628 {
1629         struct openssl_session *sess;
1630         struct openssl_qp *qp = queue_pair;
1631         int i, retval;
1632
1633         for (i = 0; i < nb_ops; i++) {
1634                 sess = get_session(qp, ops[i]);
1635                 if (unlikely(sess == NULL))
1636                         goto enqueue_err;
1637
1638                 retval = process_op(qp, ops[i], sess);
1639                 if (unlikely(retval < 0))
1640                         goto enqueue_err;
1641         }
1642
1643         qp->stats.enqueued_count += i;
1644         return i;
1645
1646 enqueue_err:
1647         qp->stats.enqueue_err_count++;
1648         return i;
1649 }
1650
1651 /** Dequeue burst */
1652 static uint16_t
1653 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1654                 uint16_t nb_ops)
1655 {
1656         struct openssl_qp *qp = queue_pair;
1657
1658         unsigned int nb_dequeued = 0;
1659
1660         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1661                         (void **)ops, nb_ops, NULL);
1662         qp->stats.dequeued_count += nb_dequeued;
1663
1664         return nb_dequeued;
1665 }
1666
1667 /** Create OPENSSL crypto device */
1668 static int
1669 cryptodev_openssl_create(const char *name,
1670                         struct rte_vdev_device *vdev,
1671                         struct rte_crypto_vdev_init_params *init_params)
1672 {
1673         struct rte_cryptodev *dev;
1674         struct openssl_private *internals;
1675
1676         if (init_params->name[0] == '\0')
1677                 snprintf(init_params->name, sizeof(init_params->name),
1678                                 "%s", name);
1679
1680         dev = rte_cryptodev_vdev_pmd_init(init_params->name,
1681                         sizeof(struct openssl_private),
1682                         init_params->socket_id,
1683                         vdev);
1684         if (dev == NULL) {
1685                 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1686                 goto init_error;
1687         }
1688
1689         dev->driver_id = cryptodev_driver_id;
1690         dev->dev_ops = rte_openssl_pmd_ops;
1691
1692         /* register rx/tx burst functions for data path */
1693         dev->dequeue_burst = openssl_pmd_dequeue_burst;
1694         dev->enqueue_burst = openssl_pmd_enqueue_burst;
1695
1696         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1697                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1698                         RTE_CRYPTODEV_FF_CPU_AESNI |
1699                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1700
1701         /* Set vector instructions mode supported */
1702         internals = dev->data->dev_private;
1703
1704         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1705         internals->max_nb_sessions = init_params->max_nb_sessions;
1706
1707         return 0;
1708
1709 init_error:
1710         OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1711                         init_params->name);
1712
1713         cryptodev_openssl_remove(vdev);
1714         return -EFAULT;
1715 }
1716
1717 /** Initialise OPENSSL crypto device */
1718 static int
1719 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1720 {
1721         struct rte_crypto_vdev_init_params init_params = {
1722                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
1723                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
1724                 rte_socket_id(),
1725                 {0}
1726         };
1727         const char *name;
1728         const char *input_args;
1729
1730         name = rte_vdev_device_name(vdev);
1731         if (name == NULL)
1732                 return -EINVAL;
1733         input_args = rte_vdev_device_args(vdev);
1734
1735         rte_cryptodev_vdev_parse_init_params(&init_params, input_args);
1736
1737         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
1738                         init_params.socket_id);
1739         if (init_params.name[0] != '\0')
1740                 RTE_LOG(INFO, PMD, "  User defined name = %s\n",
1741                         init_params.name);
1742         RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
1743                         init_params.max_nb_queue_pairs);
1744         RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
1745                         init_params.max_nb_sessions);
1746
1747         return cryptodev_openssl_create(name, vdev, &init_params);
1748 }
1749
1750 /** Uninitialise OPENSSL crypto device */
1751 static int
1752 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1753 {
1754         const char *name;
1755
1756         name = rte_vdev_device_name(vdev);
1757         if (name == NULL)
1758                 return -EINVAL;
1759
1760         RTE_LOG(INFO, PMD,
1761                 "Closing OPENSSL crypto device %s on numa socket %u\n",
1762                 name, rte_socket_id());
1763
1764         return 0;
1765 }
1766
1767 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1768         .probe = cryptodev_openssl_probe,
1769         .remove = cryptodev_openssl_remove
1770 };
1771
1772 static struct cryptodev_driver openssl_crypto_drv;
1773
1774 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1775         cryptodev_openssl_pmd_drv);
1776 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1777         "max_nb_queue_pairs=<int> "
1778         "max_nb_sessions=<int> "
1779         "socket_id=<int>");
1780 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, cryptodev_openssl_pmd_drv,
1781                 cryptodev_driver_id);