drivers/crypto/openssl/rte_openssl_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_common.h>
  34 #include <rte_hexdump.h>
  35 #include <rte_cryptodev.h>
  36 #include <rte_cryptodev_pmd.h>
  37 #include <rte_cryptodev_vdev.h>
  38 #include <rte_vdev.h>
  39 #include <rte_malloc.h>
  40 #include <rte_cpuflags.h>
  41
  42 #include <openssl/hmac.h>
  43 #include <openssl/evp.h>
  44
  45 #include "rte_openssl_pmd_private.h"
  46
  47 #define DES_BLOCK_SIZE 8
  48
  49 static uint8_t cryptodev_driver_id;
  50
  51 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  52 static HMAC_CTX *HMAC_CTX_new(void)
  53 {
  54         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  55
  56         if (ctx != NULL)
  57                 HMAC_CTX_init(ctx);
  58         return ctx;
  59 }
  60
  61 static void HMAC_CTX_free(HMAC_CTX *ctx)
  62 {
  63         if (ctx != NULL) {
  64                 HMAC_CTX_cleanup(ctx);
  65                 OPENSSL_free(ctx);
  66         }
  67 }
  68 #endif
  69
  70 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  71
  72 /*----------------------------------------------------------------------------*/
  73
  74 /**
  75  * Increment counter by 1
  76  * Counter is 64 bit array, big-endian
  77  */
  78 static void
  79 ctr_inc(uint8_t *ctr)
  80 {
  81         uint64_t *ctr64 = (uint64_t *)ctr;
  82
  83         *ctr64 = __builtin_bswap64(*ctr64);
  84         (*ctr64)++;
  85         *ctr64 = __builtin_bswap64(*ctr64);
  86 }
  87
  88 /*
  89  *------------------------------------------------------------------------------
  90  * Session Prepare
  91  *------------------------------------------------------------------------------
  92  */
  93
  94 /** Get xform chain order */
  95 static enum openssl_chain_order
  96 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
  97 {
  98         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
  99
 100         if (xform != NULL) {
 101                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 102                         if (xform->next == NULL)
 103                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
 104                         else if (xform->next->type ==
 105                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
 106                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
 107                 }
 108                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 109                         if (xform->next == NULL)
 110                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
 111                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 112                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
 113                 }
 114                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 115                         res = OPENSSL_CHAIN_COMBINED;
 116         }
 117
 118         return res;
 119 }
 120
 121 /** Get session cipher key from input cipher key */
 122 static void
 123 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
 124 {
 125         memcpy(session_key, input_key, keylen);
 126 }
 127
 128 /** Get key ede 24 bytes standard from input key */
 129 static int
 130 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
 131 {
 132         int res = 0;
 133
 134         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 135         switch (keylen) {
 136         case 24:
 137                 memcpy(key_ede, key, 24);
 138                 break;
 139         case 16:
 140                 /* K3 = K1 */
 141                 memcpy(key_ede, key, 16);
 142                 memcpy(key_ede + 16, key, 8);
 143                 break;
 144         case 8:
 145                 /* K1 = K2 = K3 (DES compatibility) */
 146                 memcpy(key_ede, key, 8);
 147                 memcpy(key_ede + 8, key, 8);
 148                 memcpy(key_ede + 16, key, 8);
 149                 break;
 150         default:
 151                 OPENSSL_LOG_ERR("Unsupported key size");
 152                 res = -EINVAL;
 153         }
 154
 155         return res;
 156 }
 157
 158 /** Get adequate openssl function for input cipher algorithm */
 159 static uint8_t
 160 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 161                 const EVP_CIPHER **algo)
 162 {
 163         int res = 0;
 164
 165         if (algo != NULL) {
 166                 switch (sess_algo) {
 167                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 168                         switch (keylen) {
 169                         case 16:
 170                                 *algo = EVP_des_ede_cbc();
 171                                 break;
 172                         case 24:
 173                                 *algo = EVP_des_ede3_cbc();
 174                                 break;
 175                         default:
 176                                 res = -EINVAL;
 177                         }
 178                         break;
 179                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 180                         break;
 181                 case RTE_CRYPTO_CIPHER_AES_CBC:
 182                         switch (keylen) {
 183                         case 16:
 184                                 *algo = EVP_aes_128_cbc();
 185                                 break;
 186                         case 24:
 187                                 *algo = EVP_aes_192_cbc();
 188                                 break;
 189                         case 32:
 190                                 *algo = EVP_aes_256_cbc();
 191                                 break;
 192                         default:
 193                                 res = -EINVAL;
 194                         }
 195                         break;
 196                 case RTE_CRYPTO_CIPHER_AES_CTR:
 197                         switch (keylen) {
 198                         case 16:
 199                                 *algo = EVP_aes_128_ctr();
 200                                 break;
 201                         case 24:
 202                                 *algo = EVP_aes_192_ctr();
 203                                 break;
 204                         case 32:
 205                                 *algo = EVP_aes_256_ctr();
 206                                 break;
 207                         default:
 208                                 res = -EINVAL;
 209                         }
 210                         break;
 211                 default:
 212                         res = -EINVAL;
 213                         break;
 214                 }
 215         } else {
 216                 res = -EINVAL;
 217         }
 218
 219         return res;
 220 }
 221
 222 /** Get adequate openssl function for input auth algorithm */
 223 static uint8_t
 224 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 225                 const EVP_MD **algo)
 226 {
 227         int res = 0;
 228
 229         if (algo != NULL) {
 230                 switch (sessalgo) {
 231                 case RTE_CRYPTO_AUTH_MD5:
 232                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 233                         *algo = EVP_md5();
 234                         break;
 235                 case RTE_CRYPTO_AUTH_SHA1:
 236                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 237                         *algo = EVP_sha1();
 238                         break;
 239                 case RTE_CRYPTO_AUTH_SHA224:
 240                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 241                         *algo = EVP_sha224();
 242                         break;
 243                 case RTE_CRYPTO_AUTH_SHA256:
 244                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 245                         *algo = EVP_sha256();
 246                         break;
 247                 case RTE_CRYPTO_AUTH_SHA384:
 248                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 249                         *algo = EVP_sha384();
 250                         break;
 251                 case RTE_CRYPTO_AUTH_SHA512:
 252                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 253                         *algo = EVP_sha512();
 254                         break;
 255                 default:
 256                         res = -EINVAL;
 257                         break;
 258                 }
 259         } else {
 260                 res = -EINVAL;
 261         }
 262
 263         return res;
 264 }
 265
 266 /** Get adequate openssl function for input cipher algorithm */
 267 static uint8_t
 268 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 269                 const EVP_CIPHER **algo)
 270 {
 271         int res = 0;
 272
 273         if (algo != NULL) {
 274                 switch (sess_algo) {
 275                 case RTE_CRYPTO_AEAD_AES_GCM:
 276                         switch (keylen) {
 277                         case 16:
 278                                 *algo = EVP_aes_128_gcm();
 279                                 break;
 280                         case 24:
 281                                 *algo = EVP_aes_192_gcm();
 282                                 break;
 283                         case 32:
 284                                 *algo = EVP_aes_256_gcm();
 285                                 break;
 286                         default:
 287                                 res = -EINVAL;
 288                         }
 289                         break;
 290                 default:
 291                         res = -EINVAL;
 292                         break;
 293                 }
 294         } else {
 295                 res = -EINVAL;
 296         }
 297
 298         return res;
 299 }
 300
 301 /** Set session cipher parameters */
 302 static int
 303 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 304                 const struct rte_crypto_sym_xform *xform)
 305 {
 306         /* Select cipher direction */
 307         sess->cipher.direction = xform->cipher.op;
 308         /* Select cipher key */
 309         sess->cipher.key.length = xform->cipher.key.length;
 310
 311         /* Set IV parameters */
 312         sess->iv.offset = xform->cipher.iv.offset;
 313         sess->iv.length = xform->cipher.iv.length;
 314
 315         /* Select cipher algo */
 316         switch (xform->cipher.algo) {
 317         case RTE_CRYPTO_CIPHER_3DES_CBC:
 318         case RTE_CRYPTO_CIPHER_AES_CBC:
 319         case RTE_CRYPTO_CIPHER_AES_CTR:
 320                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 321                 sess->cipher.algo = xform->cipher.algo;
 322                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 323
 324                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 325                                 &sess->cipher.evp_algo) != 0)
 326                         return -EINVAL;
 327
 328                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 329                         sess->cipher.key.data);
 330
 331                 break;
 332
 333         case RTE_CRYPTO_CIPHER_3DES_CTR:
 334                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 335                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 336
 337                 if (get_cipher_key_ede(xform->cipher.key.data,
 338                                 sess->cipher.key.length,
 339                                 sess->cipher.key.data) != 0)
 340                         return -EINVAL;
 341                 break;
 342         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 343                 sess->cipher.algo = xform->cipher.algo;
 344                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 345                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 346                 sess->cipher.evp_algo = EVP_des_cbc();
 347
 348                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 349                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 350                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 351                                 NULL, xform->cipher.key.data, 0) != 1)
 352                         return -EINVAL;
 353
 354                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 355                         sess->cipher.key.data);
 356                 break;
 357         default:
 358                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 359                 return -ENOTSUP;
 360         }
 361
 362         return 0;
 363 }
 364
 365 /* Set session auth parameters */
 366 static int
 367 openssl_set_session_auth_parameters(struct openssl_session *sess,
 368                 const struct rte_crypto_sym_xform *xform)
 369 {
 370         /* Select auth generate/verify */
 371         sess->auth.operation = xform->auth.op;
 372         sess->auth.algo = xform->auth.algo;
 373
 374         /* Select auth algo */
 375         switch (xform->auth.algo) {
 376         case RTE_CRYPTO_AUTH_AES_GMAC:
 377                 sess->chain_order = OPENSSL_CHAIN_COMBINED;
 378
 379                 /* Set IV parameters */
 380                 sess->iv.offset = xform->auth.iv.offset;
 381                 sess->iv.length = xform->auth.iv.length;
 382
 383                 /*
 384                  * OpenSSL requires GMAC to be a GCM operation
 385                  * with no cipher data length
 386                  */
 387                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 388                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 389                         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 390                 else
 391                         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 392
 393                 sess->cipher.key.length = xform->auth.key.length;
 394                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 395
 396                 if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM,
 397                                 sess->cipher.key.length,
 398                                 &sess->cipher.evp_algo) != 0)
 399                         return -EINVAL;
 400
 401                 get_cipher_key(xform->auth.key.data, xform->auth.key.length,
 402                         sess->cipher.key.data);
 403
 404                 break;
 405
 406         case RTE_CRYPTO_AUTH_MD5:
 407         case RTE_CRYPTO_AUTH_SHA1:
 408         case RTE_CRYPTO_AUTH_SHA224:
 409         case RTE_CRYPTO_AUTH_SHA256:
 410         case RTE_CRYPTO_AUTH_SHA384:
 411         case RTE_CRYPTO_AUTH_SHA512:
 412                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 413                 if (get_auth_algo(xform->auth.algo,
 414                                 &sess->auth.auth.evp_algo) != 0)
 415                         return -EINVAL;
 416                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 417                 break;
 418
 419         case RTE_CRYPTO_AUTH_MD5_HMAC:
 420         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 421         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 422         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 423         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 424         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 425                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 426                 sess->auth.hmac.ctx = HMAC_CTX_new();
 427                 if (get_auth_algo(xform->auth.algo,
 428                                 &sess->auth.hmac.evp_algo) != 0)
 429                         return -EINVAL;
 430
 431                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 432                                 xform->auth.key.data,
 433                                 xform->auth.key.length,
 434                                 sess->auth.hmac.evp_algo, NULL) != 1)
 435                         return -EINVAL;
 436                 break;
 437
 438         default:
 439                 return -ENOTSUP;
 440         }
 441
 442         sess->auth.digest_length = xform->auth.digest_length;
 443
 444         return 0;
 445 }
 446
 447 /* Set session AEAD parameters */
 448 static int
 449 openssl_set_session_aead_parameters(struct openssl_session *sess,
 450                 const struct rte_crypto_sym_xform *xform)
 451 {
 452         /* Select cipher direction */
 453         sess->cipher.direction = xform->cipher.op;
 454         /* Select cipher key */
 455         sess->cipher.key.length = xform->aead.key.length;
 456
 457         /* Set IV parameters */
 458         sess->iv.offset = xform->aead.iv.offset;
 459         sess->iv.length = xform->aead.iv.length;
 460
 461         /* Select auth generate/verify */
 462         sess->auth.operation = xform->auth.op;
 463         sess->auth.algo = xform->auth.algo;
 464
 465         /* Select auth algo */
 466         switch (xform->aead.algo) {
 467         case RTE_CRYPTO_AEAD_AES_GCM:
 468                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 469                 sess->aead_algo = xform->aead.algo;
 470                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 471
 472                 if (get_aead_algo(sess->aead_algo, sess->cipher.key.length,
 473                                 &sess->cipher.evp_algo) != 0)
 474                         return -EINVAL;
 475
 476                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 477                         sess->cipher.key.data);
 478
 479                 sess->chain_order = OPENSSL_CHAIN_COMBINED;
 480                 break;
 481         default:
 482                 return -ENOTSUP;
 483         }
 484
 485         sess->auth.aad_length = xform->aead.aad_length;
 486         sess->auth.digest_length = xform->aead.digest_length;
 487
 488         return 0;
 489 }
 490
 491 /** Parse crypto xform chain and set private session parameters */
 492 int
 493 openssl_set_session_parameters(struct openssl_session *sess,
 494                 const struct rte_crypto_sym_xform *xform)
 495 {
 496         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 497         const struct rte_crypto_sym_xform *auth_xform = NULL;
 498         const struct rte_crypto_sym_xform *aead_xform = NULL;
 499         int ret;
 500
 501         sess->chain_order = openssl_get_chain_order(xform);
 502         switch (sess->chain_order) {
 503         case OPENSSL_CHAIN_ONLY_CIPHER:
 504                 cipher_xform = xform;
 505                 break;
 506         case OPENSSL_CHAIN_ONLY_AUTH:
 507                 auth_xform = xform;
 508                 break;
 509         case OPENSSL_CHAIN_CIPHER_AUTH:
 510                 cipher_xform = xform;
 511                 auth_xform = xform->next;
 512                 break;
 513         case OPENSSL_CHAIN_AUTH_CIPHER:
 514                 auth_xform = xform;
 515                 cipher_xform = xform->next;
 516                 break;
 517         case OPENSSL_CHAIN_COMBINED:
 518                 aead_xform = xform;
 519                 break;
 520         default:
 521                 return -EINVAL;
 522         }
 523
 524         /* Default IV length = 0 */
 525         sess->iv.length = 0;
 526
 527         /* cipher_xform must be check before auth_xform */
 528         if (cipher_xform) {
 529                 ret = openssl_set_session_cipher_parameters(
 530                                 sess, cipher_xform);
 531                 if (ret != 0) {
 532                         OPENSSL_LOG_ERR(
 533                                 "Invalid/unsupported cipher parameters");
 534                         return ret;
 535                 }
 536         }
 537
 538         if (auth_xform) {
 539                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 540                 if (ret != 0) {
 541                         OPENSSL_LOG_ERR(
 542                                 "Invalid/unsupported auth parameters");
 543                         return ret;
 544                 }
 545         }
 546
 547         if (aead_xform) {
 548                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 549                 if (ret != 0) {
 550                         OPENSSL_LOG_ERR(
 551                                 "Invalid/unsupported AEAD parameters");
 552                         return ret;
 553                 }
 554         }
 555
 556         return 0;
 557 }
 558
 559 /** Reset private session parameters */
 560 void
 561 openssl_reset_session(struct openssl_session *sess)
 562 {
 563         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 564
 565         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 566                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 567
 568         switch (sess->auth.mode) {
 569         case OPENSSL_AUTH_AS_AUTH:
 570                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 571                 break;
 572         case OPENSSL_AUTH_AS_HMAC:
 573                 EVP_PKEY_free(sess->auth.hmac.pkey);
 574                 HMAC_CTX_free(sess->auth.hmac.ctx);
 575                 break;
 576         default:
 577                 break;
 578         }
 579 }
 580
 581 /** Provide session for operation */
 582 static struct openssl_session *
 583 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 584 {
 585         struct openssl_session *sess = NULL;
 586
 587         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 588                 /* get existing session */
 589                 if (likely(op->sym->session != NULL))
 590                         sess = (struct openssl_session *)
 591                                         get_session_private_data(
 592                                         op->sym->session,
 593                                         cryptodev_driver_id);
 594         } else {
 595                 /* provide internal session */
 596                 void *_sess = NULL;
 597                 void *_sess_private_data = NULL;
 598
 599                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 600                         return NULL;
 601
 602                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 603                         return NULL;
 604
 605                 sess = (struct openssl_session *)_sess_private_data;
 606
 607                 if (unlikely(openssl_set_session_parameters(sess,
 608                                 op->sym->xform) != 0)) {
 609                         rte_mempool_put(qp->sess_mp, _sess);
 610                         rte_mempool_put(qp->sess_mp, _sess_private_data);
 611                         sess = NULL;
 612                 }
 613                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 614                 set_session_private_data(op->sym->session, cryptodev_driver_id,
 615                         _sess_private_data);
 616         }
 617
 618         if (sess == NULL)
 619                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 620
 621         return sess;
 622 }
 623
 624 /*
 625  *------------------------------------------------------------------------------
 626  * Process Operations
 627  *------------------------------------------------------------------------------
 628  */
 629 static inline int
 630 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 631                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 632 {
 633         struct rte_mbuf *m;
 634         int dstlen;
 635         int l, n = srclen;
 636         uint8_t *src;
 637
 638         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 639                         m = m->next)
 640                 offset -= rte_pktmbuf_data_len(m);
 641
 642         if (m == 0)
 643                 return -1;
 644
 645         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 646
 647         l = rte_pktmbuf_data_len(m) - offset;
 648         if (srclen <= l) {
 649                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 650                         return -1;
 651                 *dst += l;
 652                 return 0;
 653         }
 654
 655         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 656                 return -1;
 657
 658         *dst += dstlen;
 659         n -= l;
 660
 661         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 662                 src = rte_pktmbuf_mtod(m, uint8_t *);
 663                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 664                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 665                         return -1;
 666                 *dst += dstlen;
 667                 n -= l;
 668         }
 669
 670         return 0;
 671 }
 672
 673 static inline int
 674 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 675                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 676 {
 677         struct rte_mbuf *m;
 678         int dstlen;
 679         int l, n = srclen;
 680         uint8_t *src;
 681
 682         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 683                         m = m->next)
 684                 offset -= rte_pktmbuf_data_len(m);
 685
 686         if (m == 0)
 687                 return -1;
 688
 689         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 690
 691         l = rte_pktmbuf_data_len(m) - offset;
 692         if (srclen <= l) {
 693                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 694                         return -1;
 695                 *dst += l;
 696                 return 0;
 697         }
 698
 699         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 700                 return -1;
 701
 702         *dst += dstlen;
 703         n -= l;
 704
 705         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 706                 src = rte_pktmbuf_mtod(m, uint8_t *);
 707                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 708                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 709                         return -1;
 710                 *dst += dstlen;
 711                 n -= l;
 712         }
 713
 714         return 0;
 715 }
 716
 717 /** Process standard openssl cipher encryption */
 718 static int
 719 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 720                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 721                 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
 722 {
 723         int totlen;
 724
 725         if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0)
 726                 goto process_cipher_encrypt_err;
 727
 728         EVP_CIPHER_CTX_set_padding(ctx, 0);
 729
 730         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 731                         srclen, ctx))
 732                 goto process_cipher_encrypt_err;
 733
 734         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
 735                 goto process_cipher_encrypt_err;
 736
 737         return 0;
 738
 739 process_cipher_encrypt_err:
 740         OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
 741         return -EINVAL;
 742 }
 743
 744 /** Process standard openssl cipher encryption */
 745 static int
 746 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 747                 uint8_t *iv, int srclen,
 748                 EVP_CIPHER_CTX *ctx)
 749 {
 750         uint8_t i;
 751         uint8_t encrypted_iv[DES_BLOCK_SIZE];
 752         int encrypted_ivlen;
 753
 754         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
 755                         iv, DES_BLOCK_SIZE) <= 0)
 756                 goto process_cipher_encrypt_err;
 757
 758         for (i = 0; i < srclen; i++)
 759                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
 760
 761         return 0;
 762
 763 process_cipher_encrypt_err:
 764         OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
 765         return -EINVAL;
 766 }
 767 /** Process standard openssl cipher decryption */
 768 static int
 769 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 770                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 771                 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
 772 {
 773         int totlen;
 774
 775         if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0)
 776                 goto process_cipher_decrypt_err;
 777
 778         EVP_CIPHER_CTX_set_padding(ctx, 0);
 779
 780         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 781                         srclen, ctx))
 782                 goto process_cipher_decrypt_err;
 783
 784         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
 785                 goto process_cipher_decrypt_err;
 786         return 0;
 787
 788 process_cipher_decrypt_err:
 789         OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
 790         return -EINVAL;
 791 }
 792
 793 /** Process cipher des 3 ctr encryption, decryption algorithm */
 794 static int
 795 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
 796                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 797                 EVP_CIPHER_CTX *ctx)
 798 {
 799         uint8_t ebuf[8], ctr[8];
 800         int unused, n;
 801         struct rte_mbuf *m;
 802         uint8_t *src;
 803         int l;
 804
 805         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 806                         m = m->next)
 807                 offset -= rte_pktmbuf_data_len(m);
 808
 809         if (m == 0)
 810                 goto process_cipher_des3ctr_err;
 811
 812         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 813         l = rte_pktmbuf_data_len(m) - offset;
 814
 815         /* We use 3DES encryption also for decryption.
 816          * IV is not important for 3DES ecb
 817          */
 818         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
 819                 goto process_cipher_des3ctr_err;
 820
 821         memcpy(ctr, iv, 8);
 822
 823         for (n = 0; n < srclen; n++) {
 824                 if (n % 8 == 0) {
 825                         if (EVP_EncryptUpdate(ctx,
 826                                         (unsigned char *)&ebuf, &unused,
 827                                         (const unsigned char *)&ctr, 8) <= 0)
 828                                 goto process_cipher_des3ctr_err;
 829                         ctr_inc(ctr);
 830                 }
 831                 dst[n] = *(src++) ^ ebuf[n % 8];
 832
 833                 l--;
 834                 if (!l) {
 835                         m = m->next;
 836                         if (m) {
 837                                 src = rte_pktmbuf_mtod(m, uint8_t *);
 838                                 l = rte_pktmbuf_data_len(m);
 839                         }
 840                 }
 841         }
 842
 843         return 0;
 844
 845 process_cipher_des3ctr_err:
 846         OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
 847         return -EINVAL;
 848 }
 849
 850 /** Process auth/encription aes-gcm algorithm */
 851 static int
 852 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 853                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen,
 854                 uint8_t *key, uint8_t *dst, uint8_t *tag,
 855                 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
 856 {
 857         int len = 0, unused = 0;
 858         uint8_t empty[] = {};
 859
 860         if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0)
 861                 goto process_auth_encryption_gcm_err;
 862
 863         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0)
 864                 goto process_auth_encryption_gcm_err;
 865
 866         if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0)
 867                 goto process_auth_encryption_gcm_err;
 868
 869         if (aadlen > 0)
 870                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
 871                         goto process_auth_encryption_gcm_err;
 872
 873         if (srclen > 0)
 874                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 875                                 srclen, ctx))
 876                         goto process_auth_encryption_gcm_err;
 877
 878         /* Workaround open ssl bug in version less then 1.0.1f */
 879         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
 880                 goto process_auth_encryption_gcm_err;
 881
 882         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
 883                 goto process_auth_encryption_gcm_err;
 884
 885         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
 886                 goto process_auth_encryption_gcm_err;
 887
 888         return 0;
 889
 890 process_auth_encryption_gcm_err:
 891         OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
 892         return -EINVAL;
 893 }
 894
 895 static int
 896 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 897                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen,
 898                 uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx,
 899                 const EVP_CIPHER *algo)
 900 {
 901         int len = 0, unused = 0;
 902         uint8_t empty[] = {};
 903
 904         if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0)
 905                 goto process_auth_decryption_gcm_err;
 906
 907         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0)
 908                 goto process_auth_decryption_gcm_err;
 909
 910         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
 911                 goto process_auth_decryption_gcm_err;
 912
 913         if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0)
 914                 goto process_auth_decryption_gcm_err;
 915
 916         if (aadlen > 0)
 917                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
 918                         goto process_auth_decryption_gcm_err;
 919
 920         if (srclen > 0)
 921                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 922                                 srclen, ctx))
 923                         goto process_auth_decryption_gcm_err;
 924
 925         /* Workaround open ssl bug in version less then 1.0.1f */
 926         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
 927                 goto process_auth_decryption_gcm_err;
 928
 929         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
 930                 goto process_auth_decryption_gcm_final_err;
 931
 932         return 0;
 933
 934 process_auth_decryption_gcm_err:
 935         OPENSSL_LOG_ERR("Process openssl auth description gcm failed");
 936         return -EINVAL;
 937
 938 process_auth_decryption_gcm_final_err:
 939         return -EFAULT;
 940 }
 941
 942 /** Process standard openssl auth algorithms */
 943 static int
 944 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
 945                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
 946                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
 947 {
 948         size_t dstlen;
 949         struct rte_mbuf *m;
 950         int l, n = srclen;
 951         uint8_t *src;
 952
 953         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 954                         m = m->next)
 955                 offset -= rte_pktmbuf_data_len(m);
 956
 957         if (m == 0)
 958                 goto process_auth_err;
 959
 960         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
 961                 goto process_auth_err;
 962
 963         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 964
 965         l = rte_pktmbuf_data_len(m) - offset;
 966         if (srclen <= l) {
 967                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
 968                         goto process_auth_err;
 969                 goto process_auth_final;
 970         }
 971
 972         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
 973                 goto process_auth_err;
 974
 975         n -= l;
 976
 977         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 978                 src = rte_pktmbuf_mtod(m, uint8_t *);
 979                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 980                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
 981                         goto process_auth_err;
 982                 n -= l;
 983         }
 984
 985 process_auth_final:
 986         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
 987                 goto process_auth_err;
 988         return 0;
 989
 990 process_auth_err:
 991         OPENSSL_LOG_ERR("Process openssl auth failed");
 992         return -EINVAL;
 993 }
 994
 995 /** Process standard openssl auth algorithms with hmac */
 996 static int
 997 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
 998                 int srclen, HMAC_CTX *ctx)
 999 {
1000         unsigned int dstlen;
1001         struct rte_mbuf *m;
1002         int l, n = srclen;
1003         uint8_t *src;
1004
1005         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1006                         m = m->next)
1007                 offset -= rte_pktmbuf_data_len(m);
1008
1009         if (m == 0)
1010                 goto process_auth_err;
1011
1012         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1013
1014         l = rte_pktmbuf_data_len(m) - offset;
1015         if (srclen <= l) {
1016                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1017                         goto process_auth_err;
1018                 goto process_auth_final;
1019         }
1020
1021         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1022                 goto process_auth_err;
1023
1024         n -= l;
1025
1026         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1027                 src = rte_pktmbuf_mtod(m, uint8_t *);
1028                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1029                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1030                         goto process_auth_err;
1031                 n -= l;
1032         }
1033
1034 process_auth_final:
1035         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1036                 goto process_auth_err;
1037
1038         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1039                 goto process_auth_err;
1040
1041         return 0;
1042
1043 process_auth_err:
1044         OPENSSL_LOG_ERR("Process openssl auth failed");
1045         return -EINVAL;
1046 }
1047
1048 /*----------------------------------------------------------------------------*/
1049
1050 /** Process auth/cipher combined operation */
1051 static void
1052 process_openssl_combined_op
1053                 (struct rte_crypto_op *op, struct openssl_session *sess,
1054                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1055 {
1056         /* cipher */
1057         uint8_t *dst = NULL, *iv, *tag, *aad;
1058         int srclen, ivlen, aadlen, status = -1;
1059         uint32_t offset;
1060
1061         /*
1062          * Segmented destination buffer is not supported for
1063          * encryption/decryption
1064          */
1065         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1066                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1067                 return;
1068         }
1069
1070         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1071                         sess->iv.offset);
1072         ivlen = sess->iv.length;
1073         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1074                 srclen = 0;
1075                 offset = op->sym->auth.data.offset;
1076                 aadlen = op->sym->auth.data.length;
1077                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1078                                 op->sym->auth.data.offset);
1079                 tag = op->sym->auth.digest.data;
1080                 if (tag == NULL)
1081                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1082                                 offset + aadlen);
1083         } else {
1084                 srclen = op->sym->aead.data.length;
1085                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1086                                 op->sym->aead.data.offset);
1087                 offset = op->sym->aead.data.offset;
1088                 aad = op->sym->aead.aad.data;
1089                 aadlen = sess->auth.aad_length;
1090                 tag = op->sym->aead.digest.data;
1091                 if (tag == NULL)
1092                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1093                                 offset + srclen);
1094         }
1095
1096         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1097                 status = process_openssl_auth_encryption_gcm(
1098                                 mbuf_src, offset, srclen,
1099                                 aad, aadlen, iv, ivlen, sess->cipher.key.data,
1100                                 dst, tag, sess->cipher.ctx,
1101                                 sess->cipher.evp_algo);
1102         else
1103                 status = process_openssl_auth_decryption_gcm(
1104                                 mbuf_src, offset, srclen,
1105                                 aad, aadlen, iv, ivlen, sess->cipher.key.data,
1106                                 dst, tag, sess->cipher.ctx,
1107                                 sess->cipher.evp_algo);
1108
1109         if (status != 0) {
1110                 if (status == (-EFAULT) &&
1111                                 sess->auth.operation ==
1112                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1113                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1114                 else
1115                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1116         }
1117 }
1118
1119 /** Process cipher operation */
1120 static void
1121 process_openssl_cipher_op
1122                 (struct rte_crypto_op *op, struct openssl_session *sess,
1123                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1124 {
1125         uint8_t *dst, *iv;
1126         int srclen, status;
1127
1128         /*
1129          * Segmented destination buffer is not supported for
1130          * encryption/decryption
1131          */
1132         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1133                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1134                 return;
1135         }
1136
1137         srclen = op->sym->cipher.data.length;
1138         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1139                         op->sym->cipher.data.offset);
1140
1141         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1142                         sess->iv.offset);
1143
1144         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1145                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1146                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1147                                         op->sym->cipher.data.offset, iv,
1148                                         sess->cipher.key.data, srclen,
1149                                         sess->cipher.ctx,
1150                                         sess->cipher.evp_algo);
1151                 else
1152                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1153                                         op->sym->cipher.data.offset, iv,
1154                                         sess->cipher.key.data, srclen,
1155                                         sess->cipher.ctx,
1156                                         sess->cipher.evp_algo);
1157         else
1158                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1159                                 op->sym->cipher.data.offset, iv,
1160                                 sess->cipher.key.data, srclen,
1161                                 sess->cipher.ctx);
1162
1163         if (status != 0)
1164                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1165 }
1166
1167 /** Process cipher operation */
1168 static void
1169 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1170                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1171                 struct rte_mbuf *mbuf_dst)
1172 {
1173         uint8_t *src, *dst, *iv;
1174         uint8_t block_size, last_block_len;
1175         int srclen, status = 0;
1176
1177         srclen = op->sym->cipher.data.length;
1178         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1179                         op->sym->cipher.data.offset);
1180         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1181                         op->sym->cipher.data.offset);
1182
1183         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1184                         sess->iv.offset);
1185
1186         block_size = DES_BLOCK_SIZE;
1187
1188         last_block_len = srclen % block_size;
1189         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1190                 /* Encrypt only with ECB mode XOR IV */
1191                 if (srclen < block_size) {
1192                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1193                                         iv, srclen,
1194                                         sess->cipher.bpi_ctx);
1195                 } else {
1196                         srclen -= last_block_len;
1197                         /* Encrypt with the block aligned stream with CBC mode */
1198                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1199                                         op->sym->cipher.data.offset, iv,
1200                                         sess->cipher.key.data, srclen,
1201                                         sess->cipher.ctx, sess->cipher.evp_algo);
1202                         if (last_block_len) {
1203                                 /* Point at last block */
1204                                 dst += srclen;
1205                                 /*
1206                                  * IV is the last encrypted block from
1207                                  * the previous operation
1208                                  */
1209                                 iv = dst - block_size;
1210                                 src += srclen;
1211                                 srclen = last_block_len;
1212                                 /* Encrypt the last frame with ECB mode */
1213                                 status |= process_openssl_cipher_bpi_encrypt(src,
1214                                                 dst, iv,
1215                                                 srclen, sess->cipher.bpi_ctx);
1216                         }
1217                 }
1218         } else {
1219                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1220                 if (srclen < block_size) {
1221                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1222                                         iv,
1223                                         srclen,
1224                                         sess->cipher.bpi_ctx);
1225                 } else {
1226                         if (last_block_len) {
1227                                 /* Point at last block */
1228                                 dst += srclen - last_block_len;
1229                                 src += srclen - last_block_len;
1230                                 /*
1231                                  * IV is the last full block
1232                                  */
1233                                 iv = src - block_size;
1234                                 /*
1235                                  * Decrypt the last frame with ECB mode
1236                                  * (encrypt, as it is the same operation)
1237                                  */
1238                                 status = process_openssl_cipher_bpi_encrypt(src,
1239                                                 dst, iv,
1240                                                 last_block_len, sess->cipher.bpi_ctx);
1241                                 /* Prepare parameters for CBC mode op */
1242                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1243                                                 sess->iv.offset);
1244                                 dst += last_block_len - srclen;
1245                                 srclen -= last_block_len;
1246                         }
1247
1248                         /* Decrypt with CBC mode */
1249                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1250                                         op->sym->cipher.data.offset, iv,
1251                                         sess->cipher.key.data, srclen,
1252                                         sess->cipher.ctx,
1253                                         sess->cipher.evp_algo);
1254                 }
1255         }
1256
1257         if (status != 0)
1258                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1259 }
1260
1261 /** Process auth operation */
1262 static void
1263 process_openssl_auth_op
1264                 (struct rte_crypto_op *op, struct openssl_session *sess,
1265                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1266 {
1267         uint8_t *dst;
1268         int srclen, status;
1269
1270         srclen = op->sym->auth.data.length;
1271
1272         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1273                 dst = (uint8_t *)rte_pktmbuf_append(mbuf_src,
1274                                 sess->auth.digest_length);
1275         else {
1276                 dst = op->sym->auth.digest.data;
1277                 if (dst == NULL)
1278                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1279                                         op->sym->auth.data.offset +
1280                                         op->sym->auth.data.length);
1281         }
1282
1283         switch (sess->auth.mode) {
1284         case OPENSSL_AUTH_AS_AUTH:
1285                 status = process_openssl_auth(mbuf_src, dst,
1286                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1287                                 sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1288                 break;
1289         case OPENSSL_AUTH_AS_HMAC:
1290                 status = process_openssl_auth_hmac(mbuf_src, dst,
1291                                 op->sym->auth.data.offset, srclen,
1292                                 sess->auth.hmac.ctx);
1293                 break;
1294         default:
1295                 status = -1;
1296                 break;
1297         }
1298
1299         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1300                 if (memcmp(dst, op->sym->auth.digest.data,
1301                                 sess->auth.digest_length) != 0) {
1302                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1303                 }
1304                 /* Trim area used for digest from mbuf. */
1305                 rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length);
1306         }
1307
1308         if (status != 0)
1309                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1310 }
1311
1312 /** Process crypto operation for mbuf */
1313 static int
1314 process_op(const struct openssl_qp *qp, struct rte_crypto_op *op,
1315                 struct openssl_session *sess)
1316 {
1317         struct rte_mbuf *msrc, *mdst;
1318         int retval;
1319
1320         msrc = op->sym->m_src;
1321         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1322
1323         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1324
1325         switch (sess->chain_order) {
1326         case OPENSSL_CHAIN_ONLY_CIPHER:
1327                 process_openssl_cipher_op(op, sess, msrc, mdst);
1328                 break;
1329         case OPENSSL_CHAIN_ONLY_AUTH:
1330                 process_openssl_auth_op(op, sess, msrc, mdst);
1331                 break;
1332         case OPENSSL_CHAIN_CIPHER_AUTH:
1333                 process_openssl_cipher_op(op, sess, msrc, mdst);
1334                 process_openssl_auth_op(op, sess, mdst, mdst);
1335                 break;
1336         case OPENSSL_CHAIN_AUTH_CIPHER:
1337                 process_openssl_auth_op(op, sess, msrc, mdst);
1338                 process_openssl_cipher_op(op, sess, msrc, mdst);
1339                 break;
1340         case OPENSSL_CHAIN_COMBINED:
1341                 process_openssl_combined_op(op, sess, msrc, mdst);
1342                 break;
1343         case OPENSSL_CHAIN_CIPHER_BPI:
1344                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1345                 break;
1346         default:
1347                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1348                 break;
1349         }
1350
1351         /* Free session if a session-less crypto op */
1352         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1353                 openssl_reset_session(sess);
1354                 memset(sess, 0, sizeof(struct openssl_session));
1355                 memset(op->sym->session, 0,
1356                                 rte_cryptodev_get_header_session_size());
1357                 rte_mempool_put(qp->sess_mp, sess);
1358                 rte_mempool_put(qp->sess_mp, op->sym->session);
1359                 op->sym->session = NULL;
1360         }
1361
1362         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1363                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1364
1365         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1366                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1367         else
1368                 retval = -1;
1369
1370         return retval;
1371 }
1372
1373 /*
1374  *------------------------------------------------------------------------------
1375  * PMD Framework
1376  *------------------------------------------------------------------------------
1377  */
1378
1379 /** Enqueue burst */
1380 static uint16_t
1381 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1382                 uint16_t nb_ops)
1383 {
1384         struct openssl_session *sess;
1385         struct openssl_qp *qp = queue_pair;
1386         int i, retval;
1387
1388         for (i = 0; i < nb_ops; i++) {
1389                 sess = get_session(qp, ops[i]);
1390                 if (unlikely(sess == NULL))
1391                         goto enqueue_err;
1392
1393                 retval = process_op(qp, ops[i], sess);
1394                 if (unlikely(retval < 0))
1395                         goto enqueue_err;
1396         }
1397
1398         qp->stats.enqueued_count += i;
1399         return i;
1400
1401 enqueue_err:
1402         qp->stats.enqueue_err_count++;
1403         return i;
1404 }
1405
1406 /** Dequeue burst */
1407 static uint16_t
1408 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1409                 uint16_t nb_ops)
1410 {
1411         struct openssl_qp *qp = queue_pair;
1412
1413         unsigned int nb_dequeued = 0;
1414
1415         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1416                         (void **)ops, nb_ops, NULL);
1417         qp->stats.dequeued_count += nb_dequeued;
1418
1419         return nb_dequeued;
1420 }
1421
1422 /** Create OPENSSL crypto device */
1423 static int
1424 cryptodev_openssl_create(const char *name,
1425                         struct rte_vdev_device *vdev,
1426                         struct rte_crypto_vdev_init_params *init_params)
1427 {
1428         struct rte_cryptodev *dev;
1429         struct openssl_private *internals;
1430
1431         if (init_params->name[0] == '\0')
1432                 snprintf(init_params->name, sizeof(init_params->name),
1433                                 "%s", name);
1434
1435         dev = rte_cryptodev_vdev_pmd_init(init_params->name,
1436                         sizeof(struct openssl_private),
1437                         init_params->socket_id,
1438                         vdev);
1439         if (dev == NULL) {
1440                 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1441                 goto init_error;
1442         }
1443
1444         dev->driver_id = cryptodev_driver_id;
1445         dev->dev_ops = rte_openssl_pmd_ops;
1446
1447         /* register rx/tx burst functions for data path */
1448         dev->dequeue_burst = openssl_pmd_dequeue_burst;
1449         dev->enqueue_burst = openssl_pmd_enqueue_burst;
1450
1451         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1452                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1453                         RTE_CRYPTODEV_FF_CPU_AESNI |
1454                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1455
1456         /* Set vector instructions mode supported */
1457         internals = dev->data->dev_private;
1458
1459         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1460         internals->max_nb_sessions = init_params->max_nb_sessions;
1461
1462         return 0;
1463
1464 init_error:
1465         OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1466                         init_params->name);
1467
1468         cryptodev_openssl_remove(vdev);
1469         return -EFAULT;
1470 }
1471
1472 /** Initialise OPENSSL crypto device */
1473 static int
1474 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1475 {
1476         struct rte_crypto_vdev_init_params init_params = {
1477                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
1478                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
1479                 rte_socket_id(),
1480                 {0}
1481         };
1482         const char *name;
1483         const char *input_args;
1484
1485         name = rte_vdev_device_name(vdev);
1486         if (name == NULL)
1487                 return -EINVAL;
1488         input_args = rte_vdev_device_args(vdev);
1489
1490         rte_cryptodev_vdev_parse_init_params(&init_params, input_args);
1491
1492         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
1493                         init_params.socket_id);
1494         if (init_params.name[0] != '\0')
1495                 RTE_LOG(INFO, PMD, "  User defined name = %s\n",
1496                         init_params.name);
1497         RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
1498                         init_params.max_nb_queue_pairs);
1499         RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
1500                         init_params.max_nb_sessions);
1501
1502         return cryptodev_openssl_create(name, vdev, &init_params);
1503 }
1504
1505 /** Uninitialise OPENSSL crypto device */
1506 static int
1507 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1508 {
1509         const char *name;
1510
1511         name = rte_vdev_device_name(vdev);
1512         if (name == NULL)
1513                 return -EINVAL;
1514
1515         RTE_LOG(INFO, PMD,
1516                 "Closing OPENSSL crypto device %s on numa socket %u\n",
1517                 name, rte_socket_id());
1518
1519         return 0;
1520 }
1521
1522 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1523         .probe = cryptodev_openssl_probe,
1524         .remove = cryptodev_openssl_remove
1525 };
1526
1527 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1528         cryptodev_openssl_pmd_drv);
1529 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1530         "max_nb_queue_pairs=<int> "
1531         "max_nb_sessions=<int> "
1532         "socket_id=<int>");
1533 RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_openssl_pmd_drv, cryptodev_driver_id);