drivers/crypto/openssl/rte_openssl_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_common.h>
  34 #include <rte_hexdump.h>
  35 #include <rte_cryptodev.h>
  36 #include <rte_cryptodev_pmd.h>
  37 #include <rte_cryptodev_vdev.h>
  38 #include <rte_vdev.h>
  39 #include <rte_malloc.h>
  40 #include <rte_cpuflags.h>
  41
  42 #include <openssl/hmac.h>
  43 #include <openssl/evp.h>
  44
  45 #include "rte_openssl_pmd_private.h"
  46
  47 #define DES_BLOCK_SIZE 8
  48
  49 static uint8_t cryptodev_driver_id;
  50
  51 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  52 static HMAC_CTX *HMAC_CTX_new(void)
  53 {
  54         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  55
  56         if (ctx != NULL)
  57                 HMAC_CTX_init(ctx);
  58         return ctx;
  59 }
  60
  61 static void HMAC_CTX_free(HMAC_CTX *ctx)
  62 {
  63         if (ctx != NULL) {
  64                 HMAC_CTX_cleanup(ctx);
  65                 OPENSSL_free(ctx);
  66         }
  67 }
  68 #endif
  69
  70 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  71
  72 /*----------------------------------------------------------------------------*/
  73
  74 /**
  75  * Increment counter by 1
  76  * Counter is 64 bit array, big-endian
  77  */
  78 static void
  79 ctr_inc(uint8_t *ctr)
  80 {
  81         uint64_t *ctr64 = (uint64_t *)ctr;
  82
  83         *ctr64 = __builtin_bswap64(*ctr64);
  84         (*ctr64)++;
  85         *ctr64 = __builtin_bswap64(*ctr64);
  86 }
  87
  88 /*
  89  *------------------------------------------------------------------------------
  90  * Session Prepare
  91  *------------------------------------------------------------------------------
  92  */
  93
  94 /** Get xform chain order */
  95 static enum openssl_chain_order
  96 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
  97 {
  98         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
  99
 100         if (xform != NULL) {
 101                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 102                         if (xform->next == NULL)
 103                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
 104                         else if (xform->next->type ==
 105                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
 106                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
 107                 }
 108                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 109                         if (xform->next == NULL)
 110                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
 111                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 112                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
 113                 }
 114                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 115                         res = OPENSSL_CHAIN_COMBINED;
 116         }
 117
 118         return res;
 119 }
 120
 121 /** Get session cipher key from input cipher key */
 122 static void
 123 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
 124 {
 125         memcpy(session_key, input_key, keylen);
 126 }
 127
 128 /** Get key ede 24 bytes standard from input key */
 129 static int
 130 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
 131 {
 132         int res = 0;
 133
 134         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 135         switch (keylen) {
 136         case 24:
 137                 memcpy(key_ede, key, 24);
 138                 break;
 139         case 16:
 140                 /* K3 = K1 */
 141                 memcpy(key_ede, key, 16);
 142                 memcpy(key_ede + 16, key, 8);
 143                 break;
 144         case 8:
 145                 /* K1 = K2 = K3 (DES compatibility) */
 146                 memcpy(key_ede, key, 8);
 147                 memcpy(key_ede + 8, key, 8);
 148                 memcpy(key_ede + 16, key, 8);
 149                 break;
 150         default:
 151                 OPENSSL_LOG_ERR("Unsupported key size");
 152                 res = -EINVAL;
 153         }
 154
 155         return res;
 156 }
 157
 158 /** Get adequate openssl function for input cipher algorithm */
 159 static uint8_t
 160 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 161                 const EVP_CIPHER **algo)
 162 {
 163         int res = 0;
 164
 165         if (algo != NULL) {
 166                 switch (sess_algo) {
 167                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 168                         switch (keylen) {
 169                         case 16:
 170                                 *algo = EVP_des_ede_cbc();
 171                                 break;
 172                         case 24:
 173                                 *algo = EVP_des_ede3_cbc();
 174                                 break;
 175                         default:
 176                                 res = -EINVAL;
 177                         }
 178                         break;
 179                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 180                         break;
 181                 case RTE_CRYPTO_CIPHER_AES_CBC:
 182                         switch (keylen) {
 183                         case 16:
 184                                 *algo = EVP_aes_128_cbc();
 185                                 break;
 186                         case 24:
 187                                 *algo = EVP_aes_192_cbc();
 188                                 break;
 189                         case 32:
 190                                 *algo = EVP_aes_256_cbc();
 191                                 break;
 192                         default:
 193                                 res = -EINVAL;
 194                         }
 195                         break;
 196                 case RTE_CRYPTO_CIPHER_AES_CTR:
 197                         switch (keylen) {
 198                         case 16:
 199                                 *algo = EVP_aes_128_ctr();
 200                                 break;
 201                         case 24:
 202                                 *algo = EVP_aes_192_ctr();
 203                                 break;
 204                         case 32:
 205                                 *algo = EVP_aes_256_ctr();
 206                                 break;
 207                         default:
 208                                 res = -EINVAL;
 209                         }
 210                         break;
 211                 default:
 212                         res = -EINVAL;
 213                         break;
 214                 }
 215         } else {
 216                 res = -EINVAL;
 217         }
 218
 219         return res;
 220 }
 221
 222 /** Get adequate openssl function for input auth algorithm */
 223 static uint8_t
 224 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 225                 const EVP_MD **algo)
 226 {
 227         int res = 0;
 228
 229         if (algo != NULL) {
 230                 switch (sessalgo) {
 231                 case RTE_CRYPTO_AUTH_MD5:
 232                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 233                         *algo = EVP_md5();
 234                         break;
 235                 case RTE_CRYPTO_AUTH_SHA1:
 236                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 237                         *algo = EVP_sha1();
 238                         break;
 239                 case RTE_CRYPTO_AUTH_SHA224:
 240                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 241                         *algo = EVP_sha224();
 242                         break;
 243                 case RTE_CRYPTO_AUTH_SHA256:
 244                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 245                         *algo = EVP_sha256();
 246                         break;
 247                 case RTE_CRYPTO_AUTH_SHA384:
 248                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 249                         *algo = EVP_sha384();
 250                         break;
 251                 case RTE_CRYPTO_AUTH_SHA512:
 252                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 253                         *algo = EVP_sha512();
 254                         break;
 255                 default:
 256                         res = -EINVAL;
 257                         break;
 258                 }
 259         } else {
 260                 res = -EINVAL;
 261         }
 262
 263         return res;
 264 }
 265
 266 /** Get adequate openssl function for input cipher algorithm */
 267 static uint8_t
 268 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 269                 const EVP_CIPHER **algo)
 270 {
 271         int res = 0;
 272
 273         if (algo != NULL) {
 274                 switch (sess_algo) {
 275                 case RTE_CRYPTO_AEAD_AES_GCM:
 276                         switch (keylen) {
 277                         case 16:
 278                                 *algo = EVP_aes_128_gcm();
 279                                 break;
 280                         case 24:
 281                                 *algo = EVP_aes_192_gcm();
 282                                 break;
 283                         case 32:
 284                                 *algo = EVP_aes_256_gcm();
 285                                 break;
 286                         default:
 287                                 res = -EINVAL;
 288                         }
 289                         break;
 290                 default:
 291                         res = -EINVAL;
 292                         break;
 293                 }
 294         } else {
 295                 res = -EINVAL;
 296         }
 297
 298         return res;
 299 }
 300
 301 /* Set session AEAD encryption parameters */
 302 static int
 303 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
 304                 enum rte_crypto_aead_algorithm algo,
 305                 uint8_t tag_len, uint8_t *key)
 306 {
 307         int iv_type = 0;
 308
 309         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 310         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 311
 312         /* Select AEAD algo */
 313         switch (algo) {
 314         case RTE_CRYPTO_AEAD_AES_GCM:
 315                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 316                 if (tag_len != 16)
 317                         return -EINVAL;
 318                 break;
 319         default:
 320                 return -ENOTSUP;
 321         }
 322
 323         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 324         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 325
 326         if (get_aead_algo(algo, sess->cipher.key.length,
 327                         &sess->cipher.evp_algo) != 0)
 328                 return -EINVAL;
 329
 330         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 331
 332         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 333
 334         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 335                         NULL, NULL, NULL) <= 0)
 336                 return -EINVAL;
 337
 338         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
 339                         NULL) <= 0)
 340                 return -EINVAL;
 341
 342         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 343                 return -EINVAL;
 344
 345         return 0;
 346 }
 347
 348 /* Set session AEAD decryption parameters */
 349 static int
 350 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
 351                 enum rte_crypto_aead_algorithm algo,
 352                 uint8_t tag_len, uint8_t *key)
 353 {
 354         int iv_type = 0;
 355
 356         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 357         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 358
 359         /* Select AEAD algo */
 360         switch (algo) {
 361         case RTE_CRYPTO_AEAD_AES_GCM:
 362                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 363                 if (tag_len != 16)
 364                         return -EINVAL;
 365                 break;
 366         default:
 367                 return -ENOTSUP;
 368         }
 369
 370         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 371         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 372
 373         if (get_aead_algo(algo, sess->cipher.key.length,
 374                         &sess->cipher.evp_algo) != 0)
 375                 return -EINVAL;
 376
 377         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 378
 379         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 380
 381         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 382                         NULL, NULL, NULL) <= 0)
 383                 return -EINVAL;
 384
 385         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
 386                         sess->iv.length, NULL) <= 0)
 387                 return -EINVAL;
 388
 389         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 390                 return -EINVAL;
 391
 392         return 0;
 393 }
 394
 395 /** Set session cipher parameters */
 396 static int
 397 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 398                 const struct rte_crypto_sym_xform *xform)
 399 {
 400         /* Select cipher direction */
 401         sess->cipher.direction = xform->cipher.op;
 402         /* Select cipher key */
 403         sess->cipher.key.length = xform->cipher.key.length;
 404
 405         /* Set IV parameters */
 406         sess->iv.offset = xform->cipher.iv.offset;
 407         sess->iv.length = xform->cipher.iv.length;
 408
 409         /* Select cipher algo */
 410         switch (xform->cipher.algo) {
 411         case RTE_CRYPTO_CIPHER_3DES_CBC:
 412         case RTE_CRYPTO_CIPHER_AES_CBC:
 413         case RTE_CRYPTO_CIPHER_AES_CTR:
 414                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 415                 sess->cipher.algo = xform->cipher.algo;
 416                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 417
 418                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 419                                 &sess->cipher.evp_algo) != 0)
 420                         return -EINVAL;
 421
 422                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 423                         sess->cipher.key.data);
 424                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 425                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 426                                         sess->cipher.evp_algo,
 427                                         NULL, xform->cipher.key.data,
 428                                         NULL) != 1) {
 429                                 return -EINVAL;
 430                         }
 431                 } else if (sess->cipher.direction ==
 432                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 433                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 434                                         sess->cipher.evp_algo,
 435                                         NULL, xform->cipher.key.data,
 436                                         NULL) != 1) {
 437                                 return -EINVAL;
 438                         }
 439                 }
 440
 441                 break;
 442
 443         case RTE_CRYPTO_CIPHER_3DES_CTR:
 444                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 445                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 446
 447                 if (get_cipher_key_ede(xform->cipher.key.data,
 448                                 sess->cipher.key.length,
 449                                 sess->cipher.key.data) != 0)
 450                         return -EINVAL;
 451                 break;
 452
 453         case RTE_CRYPTO_CIPHER_DES_CBC:
 454                 sess->cipher.algo = xform->cipher.algo;
 455                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 456                 sess->cipher.evp_algo = EVP_des_cbc();
 457
 458                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 459                         sess->cipher.key.data);
 460                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 461                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 462                                         sess->cipher.evp_algo,
 463                                         NULL, xform->cipher.key.data,
 464                                         NULL) != 1) {
 465                                 return -EINVAL;
 466                         }
 467                 } else if (sess->cipher.direction ==
 468                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 469                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 470                                         sess->cipher.evp_algo,
 471                                         NULL, xform->cipher.key.data,
 472                                         NULL) != 1) {
 473                                 return -EINVAL;
 474                         }
 475                 }
 476
 477                 break;
 478
 479         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 480                 sess->cipher.algo = xform->cipher.algo;
 481                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 482                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 483                 sess->cipher.evp_algo = EVP_des_cbc();
 484
 485                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 486                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 487                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 488                                 NULL, xform->cipher.key.data, 0) != 1)
 489                         return -EINVAL;
 490
 491                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 492                         sess->cipher.key.data);
 493                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 494                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 495                                         sess->cipher.evp_algo,
 496                                         NULL, xform->cipher.key.data,
 497                                         NULL) != 1) {
 498                                 return -EINVAL;
 499                         }
 500                 } else if (sess->cipher.direction ==
 501                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 502                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 503                                         sess->cipher.evp_algo,
 504                                         NULL, xform->cipher.key.data,
 505                                         NULL) != 1) {
 506                                 return -EINVAL;
 507                         }
 508                 }
 509
 510                 break;
 511         default:
 512                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 513                 return -ENOTSUP;
 514         }
 515
 516         return 0;
 517 }
 518
 519 /* Set session auth parameters */
 520 static int
 521 openssl_set_session_auth_parameters(struct openssl_session *sess,
 522                 const struct rte_crypto_sym_xform *xform)
 523 {
 524         /* Select auth generate/verify */
 525         sess->auth.operation = xform->auth.op;
 526         sess->auth.algo = xform->auth.algo;
 527
 528         sess->auth.digest_length = xform->auth.digest_length;
 529
 530         /* Select auth algo */
 531         switch (xform->auth.algo) {
 532         case RTE_CRYPTO_AUTH_AES_GMAC:
 533                 /*
 534                  * OpenSSL requires GMAC to be a GCM operation
 535                  * with no cipher data length
 536                  */
 537                 sess->cipher.key.length = xform->auth.key.length;
 538
 539                 /* Set IV parameters */
 540                 sess->iv.offset = xform->auth.iv.offset;
 541                 sess->iv.length = xform->auth.iv.length;
 542
 543                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 544                         return openssl_set_sess_aead_enc_param(sess,
 545                                                 RTE_CRYPTO_AEAD_AES_GCM,
 546                                                 xform->auth.digest_length,
 547                                                 xform->auth.key.data);
 548                 else
 549                         return openssl_set_sess_aead_dec_param(sess,
 550                                                 RTE_CRYPTO_AEAD_AES_GCM,
 551                                                 xform->auth.digest_length,
 552                                                 xform->auth.key.data);
 553                 break;
 554
 555         case RTE_CRYPTO_AUTH_MD5:
 556         case RTE_CRYPTO_AUTH_SHA1:
 557         case RTE_CRYPTO_AUTH_SHA224:
 558         case RTE_CRYPTO_AUTH_SHA256:
 559         case RTE_CRYPTO_AUTH_SHA384:
 560         case RTE_CRYPTO_AUTH_SHA512:
 561                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 562                 if (get_auth_algo(xform->auth.algo,
 563                                 &sess->auth.auth.evp_algo) != 0)
 564                         return -EINVAL;
 565                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 566                 break;
 567
 568         case RTE_CRYPTO_AUTH_MD5_HMAC:
 569         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 570         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 571         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 572         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 573         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 574                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 575                 sess->auth.hmac.ctx = HMAC_CTX_new();
 576                 if (get_auth_algo(xform->auth.algo,
 577                                 &sess->auth.hmac.evp_algo) != 0)
 578                         return -EINVAL;
 579
 580                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 581                                 xform->auth.key.data,
 582                                 xform->auth.key.length,
 583                                 sess->auth.hmac.evp_algo, NULL) != 1)
 584                         return -EINVAL;
 585                 break;
 586
 587         default:
 588                 return -ENOTSUP;
 589         }
 590
 591         return 0;
 592 }
 593
 594 /* Set session AEAD parameters */
 595 static int
 596 openssl_set_session_aead_parameters(struct openssl_session *sess,
 597                 const struct rte_crypto_sym_xform *xform)
 598 {
 599         /* Select cipher key */
 600         sess->cipher.key.length = xform->aead.key.length;
 601
 602         /* Set IV parameters */
 603         sess->iv.offset = xform->aead.iv.offset;
 604         sess->iv.length = xform->aead.iv.length;
 605
 606         sess->auth.aad_length = xform->aead.aad_length;
 607         sess->auth.digest_length = xform->aead.digest_length;
 608
 609         sess->aead_algo = xform->aead.algo;
 610         /* Select cipher direction */
 611         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 612                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
 613                                 xform->aead.digest_length, xform->aead.key.data);
 614         else
 615                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
 616                                 xform->aead.digest_length, xform->aead.key.data);
 617 }
 618
 619 /** Parse crypto xform chain and set private session parameters */
 620 int
 621 openssl_set_session_parameters(struct openssl_session *sess,
 622                 const struct rte_crypto_sym_xform *xform)
 623 {
 624         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 625         const struct rte_crypto_sym_xform *auth_xform = NULL;
 626         const struct rte_crypto_sym_xform *aead_xform = NULL;
 627         int ret;
 628
 629         sess->chain_order = openssl_get_chain_order(xform);
 630         switch (sess->chain_order) {
 631         case OPENSSL_CHAIN_ONLY_CIPHER:
 632                 cipher_xform = xform;
 633                 break;
 634         case OPENSSL_CHAIN_ONLY_AUTH:
 635                 auth_xform = xform;
 636                 break;
 637         case OPENSSL_CHAIN_CIPHER_AUTH:
 638                 cipher_xform = xform;
 639                 auth_xform = xform->next;
 640                 break;
 641         case OPENSSL_CHAIN_AUTH_CIPHER:
 642                 auth_xform = xform;
 643                 cipher_xform = xform->next;
 644                 break;
 645         case OPENSSL_CHAIN_COMBINED:
 646                 aead_xform = xform;
 647                 break;
 648         default:
 649                 return -EINVAL;
 650         }
 651
 652         /* Default IV length = 0 */
 653         sess->iv.length = 0;
 654
 655         /* cipher_xform must be check before auth_xform */
 656         if (cipher_xform) {
 657                 ret = openssl_set_session_cipher_parameters(
 658                                 sess, cipher_xform);
 659                 if (ret != 0) {
 660                         OPENSSL_LOG_ERR(
 661                                 "Invalid/unsupported cipher parameters");
 662                         return ret;
 663                 }
 664         }
 665
 666         if (auth_xform) {
 667                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 668                 if (ret != 0) {
 669                         OPENSSL_LOG_ERR(
 670                                 "Invalid/unsupported auth parameters");
 671                         return ret;
 672                 }
 673         }
 674
 675         if (aead_xform) {
 676                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 677                 if (ret != 0) {
 678                         OPENSSL_LOG_ERR(
 679                                 "Invalid/unsupported AEAD parameters");
 680                         return ret;
 681                 }
 682         }
 683
 684         return 0;
 685 }
 686
 687 /** Reset private session parameters */
 688 void
 689 openssl_reset_session(struct openssl_session *sess)
 690 {
 691         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 692
 693         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 694                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 695
 696         switch (sess->auth.mode) {
 697         case OPENSSL_AUTH_AS_AUTH:
 698                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 699                 break;
 700         case OPENSSL_AUTH_AS_HMAC:
 701                 EVP_PKEY_free(sess->auth.hmac.pkey);
 702                 HMAC_CTX_free(sess->auth.hmac.ctx);
 703                 break;
 704         default:
 705                 break;
 706         }
 707 }
 708
 709 /** Provide session for operation */
 710 static struct openssl_session *
 711 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 712 {
 713         struct openssl_session *sess = NULL;
 714
 715         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 716                 /* get existing session */
 717                 if (likely(op->sym->session != NULL))
 718                         sess = (struct openssl_session *)
 719                                         get_session_private_data(
 720                                         op->sym->session,
 721                                         cryptodev_driver_id);
 722         } else {
 723                 /* provide internal session */
 724                 void *_sess = NULL;
 725                 void *_sess_private_data = NULL;
 726
 727                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 728                         return NULL;
 729
 730                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 731                         return NULL;
 732
 733                 sess = (struct openssl_session *)_sess_private_data;
 734
 735                 if (unlikely(openssl_set_session_parameters(sess,
 736                                 op->sym->xform) != 0)) {
 737                         rte_mempool_put(qp->sess_mp, _sess);
 738                         rte_mempool_put(qp->sess_mp, _sess_private_data);
 739                         sess = NULL;
 740                 }
 741                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 742                 set_session_private_data(op->sym->session, cryptodev_driver_id,
 743                         _sess_private_data);
 744         }
 745
 746         if (sess == NULL)
 747                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 748
 749         return sess;
 750 }
 751
 752 /*
 753  *------------------------------------------------------------------------------
 754  * Process Operations
 755  *------------------------------------------------------------------------------
 756  */
 757 static inline int
 758 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 759                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 760 {
 761         struct rte_mbuf *m;
 762         int dstlen;
 763         int l, n = srclen;
 764         uint8_t *src;
 765
 766         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 767                         m = m->next)
 768                 offset -= rte_pktmbuf_data_len(m);
 769
 770         if (m == 0)
 771                 return -1;
 772
 773         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 774
 775         l = rte_pktmbuf_data_len(m) - offset;
 776         if (srclen <= l) {
 777                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 778                         return -1;
 779                 *dst += l;
 780                 return 0;
 781         }
 782
 783         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 784                 return -1;
 785
 786         *dst += dstlen;
 787         n -= l;
 788
 789         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 790                 src = rte_pktmbuf_mtod(m, uint8_t *);
 791                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 792                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 793                         return -1;
 794                 *dst += dstlen;
 795                 n -= l;
 796         }
 797
 798         return 0;
 799 }
 800
 801 static inline int
 802 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 803                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 804 {
 805         struct rte_mbuf *m;
 806         int dstlen;
 807         int l, n = srclen;
 808         uint8_t *src;
 809
 810         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 811                         m = m->next)
 812                 offset -= rte_pktmbuf_data_len(m);
 813
 814         if (m == 0)
 815                 return -1;
 816
 817         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 818
 819         l = rte_pktmbuf_data_len(m) - offset;
 820         if (srclen <= l) {
 821                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 822                         return -1;
 823                 *dst += l;
 824                 return 0;
 825         }
 826
 827         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 828                 return -1;
 829
 830         *dst += dstlen;
 831         n -= l;
 832
 833         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 834                 src = rte_pktmbuf_mtod(m, uint8_t *);
 835                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 836                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 837                         return -1;
 838                 *dst += dstlen;
 839                 n -= l;
 840         }
 841
 842         return 0;
 843 }
 844
 845 /** Process standard openssl cipher encryption */
 846 static int
 847 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 848                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 849 {
 850         int totlen;
 851
 852         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 853                 goto process_cipher_encrypt_err;
 854
 855         EVP_CIPHER_CTX_set_padding(ctx, 0);
 856
 857         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 858                         srclen, ctx))
 859                 goto process_cipher_encrypt_err;
 860
 861         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
 862                 goto process_cipher_encrypt_err;
 863
 864         return 0;
 865
 866 process_cipher_encrypt_err:
 867         OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
 868         return -EINVAL;
 869 }
 870
 871 /** Process standard openssl cipher encryption */
 872 static int
 873 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 874                 uint8_t *iv, int srclen,
 875                 EVP_CIPHER_CTX *ctx)
 876 {
 877         uint8_t i;
 878         uint8_t encrypted_iv[DES_BLOCK_SIZE];
 879         int encrypted_ivlen;
 880
 881         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
 882                         iv, DES_BLOCK_SIZE) <= 0)
 883                 goto process_cipher_encrypt_err;
 884
 885         for (i = 0; i < srclen; i++)
 886                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
 887
 888         return 0;
 889
 890 process_cipher_encrypt_err:
 891         OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
 892         return -EINVAL;
 893 }
 894 /** Process standard openssl cipher decryption */
 895 static int
 896 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 897                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 898 {
 899         int totlen;
 900
 901         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 902                 goto process_cipher_decrypt_err;
 903
 904         EVP_CIPHER_CTX_set_padding(ctx, 0);
 905
 906         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 907                         srclen, ctx))
 908                 goto process_cipher_decrypt_err;
 909
 910         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
 911                 goto process_cipher_decrypt_err;
 912         return 0;
 913
 914 process_cipher_decrypt_err:
 915         OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
 916         return -EINVAL;
 917 }
 918
 919 /** Process cipher des 3 ctr encryption, decryption algorithm */
 920 static int
 921 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
 922                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 923                 EVP_CIPHER_CTX *ctx)
 924 {
 925         uint8_t ebuf[8], ctr[8];
 926         int unused, n;
 927         struct rte_mbuf *m;
 928         uint8_t *src;
 929         int l;
 930
 931         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 932                         m = m->next)
 933                 offset -= rte_pktmbuf_data_len(m);
 934
 935         if (m == 0)
 936                 goto process_cipher_des3ctr_err;
 937
 938         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 939         l = rte_pktmbuf_data_len(m) - offset;
 940
 941         /* We use 3DES encryption also for decryption.
 942          * IV is not important for 3DES ecb
 943          */
 944         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
 945                 goto process_cipher_des3ctr_err;
 946
 947         memcpy(ctr, iv, 8);
 948
 949         for (n = 0; n < srclen; n++) {
 950                 if (n % 8 == 0) {
 951                         if (EVP_EncryptUpdate(ctx,
 952                                         (unsigned char *)&ebuf, &unused,
 953                                         (const unsigned char *)&ctr, 8) <= 0)
 954                                 goto process_cipher_des3ctr_err;
 955                         ctr_inc(ctr);
 956                 }
 957                 dst[n] = *(src++) ^ ebuf[n % 8];
 958
 959                 l--;
 960                 if (!l) {
 961                         m = m->next;
 962                         if (m) {
 963                                 src = rte_pktmbuf_mtod(m, uint8_t *);
 964                                 l = rte_pktmbuf_data_len(m);
 965                         }
 966                 }
 967         }
 968
 969         return 0;
 970
 971 process_cipher_des3ctr_err:
 972         OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
 973         return -EINVAL;
 974 }
 975
 976 /** Process auth/encription aes-gcm algorithm */
 977 static int
 978 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 979                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
 980                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
 981 {
 982         int len = 0, unused = 0;
 983         uint8_t empty[] = {};
 984
 985         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 986                 goto process_auth_encryption_gcm_err;
 987
 988         if (aadlen > 0)
 989                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
 990                         goto process_auth_encryption_gcm_err;
 991
 992         if (srclen > 0)
 993                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 994                                 srclen, ctx))
 995                         goto process_auth_encryption_gcm_err;
 996
 997         /* Workaround open ssl bug in version less then 1.0.1f */
 998         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
 999                 goto process_auth_encryption_gcm_err;
1000
1001         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1002                 goto process_auth_encryption_gcm_err;
1003
1004         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1005                 goto process_auth_encryption_gcm_err;
1006
1007         return 0;
1008
1009 process_auth_encryption_gcm_err:
1010         OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
1011         return -EINVAL;
1012 }
1013
1014 static int
1015 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1016                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1017                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1018 {
1019         int len = 0, unused = 0;
1020         uint8_t empty[] = {};
1021
1022         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1023                 goto process_auth_decryption_gcm_err;
1024
1025         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1026                 goto process_auth_decryption_gcm_err;
1027
1028         if (aadlen > 0)
1029                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1030                         goto process_auth_decryption_gcm_err;
1031
1032         if (srclen > 0)
1033                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1034                                 srclen, ctx))
1035                         goto process_auth_decryption_gcm_err;
1036
1037         /* Workaround open ssl bug in version less then 1.0.1f */
1038         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1039                 goto process_auth_decryption_gcm_err;
1040
1041         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1042                 goto process_auth_decryption_gcm_final_err;
1043
1044         return 0;
1045
1046 process_auth_decryption_gcm_err:
1047         OPENSSL_LOG_ERR("Process openssl auth description gcm failed");
1048         return -EINVAL;
1049
1050 process_auth_decryption_gcm_final_err:
1051         return -EFAULT;
1052 }
1053
1054 /** Process standard openssl auth algorithms */
1055 static int
1056 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1057                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1058                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1059 {
1060         size_t dstlen;
1061         struct rte_mbuf *m;
1062         int l, n = srclen;
1063         uint8_t *src;
1064
1065         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1066                         m = m->next)
1067                 offset -= rte_pktmbuf_data_len(m);
1068
1069         if (m == 0)
1070                 goto process_auth_err;
1071
1072         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1073                 goto process_auth_err;
1074
1075         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1076
1077         l = rte_pktmbuf_data_len(m) - offset;
1078         if (srclen <= l) {
1079                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1080                         goto process_auth_err;
1081                 goto process_auth_final;
1082         }
1083
1084         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1085                 goto process_auth_err;
1086
1087         n -= l;
1088
1089         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1090                 src = rte_pktmbuf_mtod(m, uint8_t *);
1091                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1092                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1093                         goto process_auth_err;
1094                 n -= l;
1095         }
1096
1097 process_auth_final:
1098         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1099                 goto process_auth_err;
1100         return 0;
1101
1102 process_auth_err:
1103         OPENSSL_LOG_ERR("Process openssl auth failed");
1104         return -EINVAL;
1105 }
1106
1107 /** Process standard openssl auth algorithms with hmac */
1108 static int
1109 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1110                 int srclen, HMAC_CTX *ctx)
1111 {
1112         unsigned int dstlen;
1113         struct rte_mbuf *m;
1114         int l, n = srclen;
1115         uint8_t *src;
1116
1117         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1118                         m = m->next)
1119                 offset -= rte_pktmbuf_data_len(m);
1120
1121         if (m == 0)
1122                 goto process_auth_err;
1123
1124         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1125
1126         l = rte_pktmbuf_data_len(m) - offset;
1127         if (srclen <= l) {
1128                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1129                         goto process_auth_err;
1130                 goto process_auth_final;
1131         }
1132
1133         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1134                 goto process_auth_err;
1135
1136         n -= l;
1137
1138         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1139                 src = rte_pktmbuf_mtod(m, uint8_t *);
1140                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1141                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1142                         goto process_auth_err;
1143                 n -= l;
1144         }
1145
1146 process_auth_final:
1147         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1148                 goto process_auth_err;
1149
1150         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1151                 goto process_auth_err;
1152
1153         return 0;
1154
1155 process_auth_err:
1156         OPENSSL_LOG_ERR("Process openssl auth failed");
1157         return -EINVAL;
1158 }
1159
1160 /*----------------------------------------------------------------------------*/
1161
1162 /** Process auth/cipher combined operation */
1163 static void
1164 process_openssl_combined_op
1165                 (struct rte_crypto_op *op, struct openssl_session *sess,
1166                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1167 {
1168         /* cipher */
1169         uint8_t *dst = NULL, *iv, *tag, *aad;
1170         int srclen, aadlen, status = -1;
1171         uint32_t offset;
1172
1173         /*
1174          * Segmented destination buffer is not supported for
1175          * encryption/decryption
1176          */
1177         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1178                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1179                 return;
1180         }
1181
1182         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1183                         sess->iv.offset);
1184         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1185                 srclen = 0;
1186                 offset = op->sym->auth.data.offset;
1187                 aadlen = op->sym->auth.data.length;
1188                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1189                                 op->sym->auth.data.offset);
1190                 tag = op->sym->auth.digest.data;
1191                 if (tag == NULL)
1192                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1193                                 offset + aadlen);
1194         } else {
1195                 srclen = op->sym->aead.data.length;
1196                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1197                                 op->sym->aead.data.offset);
1198                 offset = op->sym->aead.data.offset;
1199                 aad = op->sym->aead.aad.data;
1200                 aadlen = sess->auth.aad_length;
1201                 tag = op->sym->aead.digest.data;
1202                 if (tag == NULL)
1203                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1204                                 offset + srclen);
1205         }
1206
1207         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1208                 status = process_openssl_auth_encryption_gcm(
1209                                 mbuf_src, offset, srclen,
1210                                 aad, aadlen, iv,
1211                                 dst, tag, sess->cipher.ctx);
1212         else
1213                 status = process_openssl_auth_decryption_gcm(
1214                                 mbuf_src, offset, srclen,
1215                                 aad, aadlen, iv,
1216                                 dst, tag, sess->cipher.ctx);
1217
1218         if (status != 0) {
1219                 if (status == (-EFAULT) &&
1220                                 sess->auth.operation ==
1221                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1222                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1223                 else
1224                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1225         }
1226 }
1227
1228 /** Process cipher operation */
1229 static void
1230 process_openssl_cipher_op
1231                 (struct rte_crypto_op *op, struct openssl_session *sess,
1232                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1233 {
1234         uint8_t *dst, *iv;
1235         int srclen, status;
1236
1237         /*
1238          * Segmented destination buffer is not supported for
1239          * encryption/decryption
1240          */
1241         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1242                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1243                 return;
1244         }
1245
1246         srclen = op->sym->cipher.data.length;
1247         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1248                         op->sym->cipher.data.offset);
1249
1250         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1251                         sess->iv.offset);
1252
1253         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1254                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1255                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1256                                         op->sym->cipher.data.offset, iv,
1257                                         srclen, sess->cipher.ctx);
1258                 else
1259                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1260                                         op->sym->cipher.data.offset, iv,
1261                                         srclen, sess->cipher.ctx);
1262         else
1263                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1264                                 op->sym->cipher.data.offset, iv,
1265                                 sess->cipher.key.data, srclen,
1266                                 sess->cipher.ctx);
1267
1268         if (status != 0)
1269                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1270 }
1271
1272 /** Process cipher operation */
1273 static void
1274 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1275                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1276                 struct rte_mbuf *mbuf_dst)
1277 {
1278         uint8_t *src, *dst, *iv;
1279         uint8_t block_size, last_block_len;
1280         int srclen, status = 0;
1281
1282         srclen = op->sym->cipher.data.length;
1283         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1284                         op->sym->cipher.data.offset);
1285         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1286                         op->sym->cipher.data.offset);
1287
1288         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1289                         sess->iv.offset);
1290
1291         block_size = DES_BLOCK_SIZE;
1292
1293         last_block_len = srclen % block_size;
1294         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1295                 /* Encrypt only with ECB mode XOR IV */
1296                 if (srclen < block_size) {
1297                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1298                                         iv, srclen,
1299                                         sess->cipher.bpi_ctx);
1300                 } else {
1301                         srclen -= last_block_len;
1302                         /* Encrypt with the block aligned stream with CBC mode */
1303                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1304                                         op->sym->cipher.data.offset, iv,
1305                                         srclen, sess->cipher.ctx);
1306                         if (last_block_len) {
1307                                 /* Point at last block */
1308                                 dst += srclen;
1309                                 /*
1310                                  * IV is the last encrypted block from
1311                                  * the previous operation
1312                                  */
1313                                 iv = dst - block_size;
1314                                 src += srclen;
1315                                 srclen = last_block_len;
1316                                 /* Encrypt the last frame with ECB mode */
1317                                 status |= process_openssl_cipher_bpi_encrypt(src,
1318                                                 dst, iv,
1319                                                 srclen, sess->cipher.bpi_ctx);
1320                         }
1321                 }
1322         } else {
1323                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1324                 if (srclen < block_size) {
1325                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1326                                         iv,
1327                                         srclen,
1328                                         sess->cipher.bpi_ctx);
1329                 } else {
1330                         if (last_block_len) {
1331                                 /* Point at last block */
1332                                 dst += srclen - last_block_len;
1333                                 src += srclen - last_block_len;
1334                                 /*
1335                                  * IV is the last full block
1336                                  */
1337                                 iv = src - block_size;
1338                                 /*
1339                                  * Decrypt the last frame with ECB mode
1340                                  * (encrypt, as it is the same operation)
1341                                  */
1342                                 status = process_openssl_cipher_bpi_encrypt(src,
1343                                                 dst, iv,
1344                                                 last_block_len, sess->cipher.bpi_ctx);
1345                                 /* Prepare parameters for CBC mode op */
1346                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1347                                                 sess->iv.offset);
1348                                 dst += last_block_len - srclen;
1349                                 srclen -= last_block_len;
1350                         }
1351
1352                         /* Decrypt with CBC mode */
1353                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1354                                         op->sym->cipher.data.offset, iv,
1355                                         srclen, sess->cipher.ctx);
1356                 }
1357         }
1358
1359         if (status != 0)
1360                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1361 }
1362
1363 /** Process auth operation */
1364 static void
1365 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1366                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1367                 struct rte_mbuf *mbuf_dst)
1368 {
1369         uint8_t *dst;
1370         int srclen, status;
1371
1372         srclen = op->sym->auth.data.length;
1373
1374         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1375                 dst = qp->temp_digest;
1376         else {
1377                 dst = op->sym->auth.digest.data;
1378                 if (dst == NULL)
1379                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1380                                         op->sym->auth.data.offset +
1381                                         op->sym->auth.data.length);
1382         }
1383
1384         switch (sess->auth.mode) {
1385         case OPENSSL_AUTH_AS_AUTH:
1386                 status = process_openssl_auth(mbuf_src, dst,
1387                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1388                                 sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1389                 break;
1390         case OPENSSL_AUTH_AS_HMAC:
1391                 status = process_openssl_auth_hmac(mbuf_src, dst,
1392                                 op->sym->auth.data.offset, srclen,
1393                                 sess->auth.hmac.ctx);
1394                 break;
1395         default:
1396                 status = -1;
1397                 break;
1398         }
1399
1400         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1401                 if (memcmp(dst, op->sym->auth.digest.data,
1402                                 sess->auth.digest_length) != 0) {
1403                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1404                 }
1405         }
1406
1407         if (status != 0)
1408                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1409 }
1410
1411 /** Process crypto operation for mbuf */
1412 static int
1413 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1414                 struct openssl_session *sess)
1415 {
1416         struct rte_mbuf *msrc, *mdst;
1417         int retval;
1418
1419         msrc = op->sym->m_src;
1420         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1421
1422         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1423
1424         switch (sess->chain_order) {
1425         case OPENSSL_CHAIN_ONLY_CIPHER:
1426                 process_openssl_cipher_op(op, sess, msrc, mdst);
1427                 break;
1428         case OPENSSL_CHAIN_ONLY_AUTH:
1429                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1430                 break;
1431         case OPENSSL_CHAIN_CIPHER_AUTH:
1432                 process_openssl_cipher_op(op, sess, msrc, mdst);
1433                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
1434                 break;
1435         case OPENSSL_CHAIN_AUTH_CIPHER:
1436                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1437                 process_openssl_cipher_op(op, sess, msrc, mdst);
1438                 break;
1439         case OPENSSL_CHAIN_COMBINED:
1440                 process_openssl_combined_op(op, sess, msrc, mdst);
1441                 break;
1442         case OPENSSL_CHAIN_CIPHER_BPI:
1443                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1444                 break;
1445         default:
1446                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1447                 break;
1448         }
1449
1450         /* Free session if a session-less crypto op */
1451         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1452                 openssl_reset_session(sess);
1453                 memset(sess, 0, sizeof(struct openssl_session));
1454                 memset(op->sym->session, 0,
1455                                 rte_cryptodev_get_header_session_size());
1456                 rte_mempool_put(qp->sess_mp, sess);
1457                 rte_mempool_put(qp->sess_mp, op->sym->session);
1458                 op->sym->session = NULL;
1459         }
1460
1461         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1462                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1463
1464         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1465                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1466         else
1467                 retval = -1;
1468
1469         return retval;
1470 }
1471
1472 /*
1473  *------------------------------------------------------------------------------
1474  * PMD Framework
1475  *------------------------------------------------------------------------------
1476  */
1477
1478 /** Enqueue burst */
1479 static uint16_t
1480 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1481                 uint16_t nb_ops)
1482 {
1483         struct openssl_session *sess;
1484         struct openssl_qp *qp = queue_pair;
1485         int i, retval;
1486
1487         for (i = 0; i < nb_ops; i++) {
1488                 sess = get_session(qp, ops[i]);
1489                 if (unlikely(sess == NULL))
1490                         goto enqueue_err;
1491
1492                 retval = process_op(qp, ops[i], sess);
1493                 if (unlikely(retval < 0))
1494                         goto enqueue_err;
1495         }
1496
1497         qp->stats.enqueued_count += i;
1498         return i;
1499
1500 enqueue_err:
1501         qp->stats.enqueue_err_count++;
1502         return i;
1503 }
1504
1505 /** Dequeue burst */
1506 static uint16_t
1507 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1508                 uint16_t nb_ops)
1509 {
1510         struct openssl_qp *qp = queue_pair;
1511
1512         unsigned int nb_dequeued = 0;
1513
1514         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1515                         (void **)ops, nb_ops, NULL);
1516         qp->stats.dequeued_count += nb_dequeued;
1517
1518         return nb_dequeued;
1519 }
1520
1521 /** Create OPENSSL crypto device */
1522 static int
1523 cryptodev_openssl_create(const char *name,
1524                         struct rte_vdev_device *vdev,
1525                         struct rte_crypto_vdev_init_params *init_params)
1526 {
1527         struct rte_cryptodev *dev;
1528         struct openssl_private *internals;
1529
1530         if (init_params->name[0] == '\0')
1531                 snprintf(init_params->name, sizeof(init_params->name),
1532                                 "%s", name);
1533
1534         dev = rte_cryptodev_vdev_pmd_init(init_params->name,
1535                         sizeof(struct openssl_private),
1536                         init_params->socket_id,
1537                         vdev);
1538         if (dev == NULL) {
1539                 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1540                 goto init_error;
1541         }
1542
1543         dev->driver_id = cryptodev_driver_id;
1544         dev->dev_ops = rte_openssl_pmd_ops;
1545
1546         /* register rx/tx burst functions for data path */
1547         dev->dequeue_burst = openssl_pmd_dequeue_burst;
1548         dev->enqueue_burst = openssl_pmd_enqueue_burst;
1549
1550         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1551                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1552                         RTE_CRYPTODEV_FF_CPU_AESNI |
1553                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1554
1555         /* Set vector instructions mode supported */
1556         internals = dev->data->dev_private;
1557
1558         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1559         internals->max_nb_sessions = init_params->max_nb_sessions;
1560
1561         return 0;
1562
1563 init_error:
1564         OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1565                         init_params->name);
1566
1567         cryptodev_openssl_remove(vdev);
1568         return -EFAULT;
1569 }
1570
1571 /** Initialise OPENSSL crypto device */
1572 static int
1573 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1574 {
1575         struct rte_crypto_vdev_init_params init_params = {
1576                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
1577                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
1578                 rte_socket_id(),
1579                 {0}
1580         };
1581         const char *name;
1582         const char *input_args;
1583
1584         name = rte_vdev_device_name(vdev);
1585         if (name == NULL)
1586                 return -EINVAL;
1587         input_args = rte_vdev_device_args(vdev);
1588
1589         rte_cryptodev_vdev_parse_init_params(&init_params, input_args);
1590
1591         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
1592                         init_params.socket_id);
1593         if (init_params.name[0] != '\0')
1594                 RTE_LOG(INFO, PMD, "  User defined name = %s\n",
1595                         init_params.name);
1596         RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
1597                         init_params.max_nb_queue_pairs);
1598         RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
1599                         init_params.max_nb_sessions);
1600
1601         return cryptodev_openssl_create(name, vdev, &init_params);
1602 }
1603
1604 /** Uninitialise OPENSSL crypto device */
1605 static int
1606 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1607 {
1608         const char *name;
1609
1610         name = rte_vdev_device_name(vdev);
1611         if (name == NULL)
1612                 return -EINVAL;
1613
1614         RTE_LOG(INFO, PMD,
1615                 "Closing OPENSSL crypto device %s on numa socket %u\n",
1616                 name, rte_socket_id());
1617
1618         return 0;
1619 }
1620
1621 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1622         .probe = cryptodev_openssl_probe,
1623         .remove = cryptodev_openssl_remove
1624 };
1625
1626 static struct cryptodev_driver openssl_crypto_drv;
1627
1628 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1629         cryptodev_openssl_pmd_drv);
1630 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1631         "max_nb_queue_pairs=<int> "
1632         "max_nb_sessions=<int> "
1633         "socket_id=<int>");
1634 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, cryptodev_openssl_pmd_drv,
1635                 cryptodev_driver_id);