1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2017 Intel Corporation
5 #include <rte_common.h>
6 #include <rte_hexdump.h>
7 #include <rte_cryptodev.h>
8 #include <cryptodev_pmd.h>
9 #include <rte_bus_vdev.h>
10 #include <rte_malloc.h>
11 #include <rte_cpuflags.h>
13 #include <openssl/hmac.h>
14 #include <openssl/evp.h>
16 #include "openssl_pmd_private.h"
19 #define DES_BLOCK_SIZE 8
21 static uint8_t cryptodev_driver_id;
23 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
24 static HMAC_CTX *HMAC_CTX_new(void)
26 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
33 static void HMAC_CTX_free(HMAC_CTX *ctx)
36 HMAC_CTX_cleanup(ctx);
42 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
44 #include <openssl/provider.h>
45 #include <openssl/core_names.h>
46 #include <openssl/param_build.h>
48 #define MAX_OSSL_ALGO_NAME_SIZE 16
50 OSSL_PROVIDER *legacy;
53 static void ossl_legacy_provider_load(void)
55 /* Load Multiple providers into the default (NULL) library context */
56 legacy = OSSL_PROVIDER_load(NULL, "legacy");
58 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n");
62 deflt = OSSL_PROVIDER_load(NULL, "default");
64 OPENSSL_LOG(ERR, "Failed to load Default provider\n");
65 OSSL_PROVIDER_unload(legacy);
70 static void ossl_legacy_provider_unload(void)
72 OSSL_PROVIDER_unload(legacy);
73 OSSL_PROVIDER_unload(deflt);
76 static __rte_always_inline const char *
77 digest_name_get(enum rte_crypto_auth_algorithm algo)
80 case RTE_CRYPTO_AUTH_MD5_HMAC:
81 return OSSL_DIGEST_NAME_MD5;
82 case RTE_CRYPTO_AUTH_SHA1_HMAC:
83 return OSSL_DIGEST_NAME_SHA1;
84 case RTE_CRYPTO_AUTH_SHA224_HMAC:
85 return OSSL_DIGEST_NAME_SHA2_224;
86 case RTE_CRYPTO_AUTH_SHA256_HMAC:
87 return OSSL_DIGEST_NAME_SHA2_256;
88 case RTE_CRYPTO_AUTH_SHA384_HMAC:
89 return OSSL_DIGEST_NAME_SHA2_384;
90 case RTE_CRYPTO_AUTH_SHA512_HMAC:
91 return OSSL_DIGEST_NAME_SHA2_512;
98 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
100 /*----------------------------------------------------------------------------*/
103 * Increment counter by 1
104 * Counter is 64 bit array, big-endian
107 ctr_inc(uint8_t *ctr)
109 uint64_t *ctr64 = (uint64_t *)ctr;
111 *ctr64 = __builtin_bswap64(*ctr64);
113 *ctr64 = __builtin_bswap64(*ctr64);
117 *------------------------------------------------------------------------------
119 *------------------------------------------------------------------------------
122 /** Get xform chain order */
123 static enum openssl_chain_order
124 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
126 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
129 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
130 if (xform->next == NULL)
131 res = OPENSSL_CHAIN_ONLY_AUTH;
132 else if (xform->next->type ==
133 RTE_CRYPTO_SYM_XFORM_CIPHER)
134 res = OPENSSL_CHAIN_AUTH_CIPHER;
136 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
137 if (xform->next == NULL)
138 res = OPENSSL_CHAIN_ONLY_CIPHER;
139 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
140 res = OPENSSL_CHAIN_CIPHER_AUTH;
142 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
143 res = OPENSSL_CHAIN_COMBINED;
149 /** Get session cipher key from input cipher key */
151 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key)
153 memcpy(session_key, input_key, keylen);
156 /** Get key ede 24 bytes standard from input key */
158 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede)
162 /* Initialize keys - 24 bytes: [key1-key2-key3] */
165 memcpy(key_ede, key, 24);
169 memcpy(key_ede, key, 16);
170 memcpy(key_ede + 16, key, 8);
173 /* K1 = K2 = K3 (DES compatibility) */
174 memcpy(key_ede, key, 8);
175 memcpy(key_ede + 8, key, 8);
176 memcpy(key_ede + 16, key, 8);
179 OPENSSL_LOG(ERR, "Unsupported key size");
186 /** Get adequate openssl function for input cipher algorithm */
188 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
189 const EVP_CIPHER **algo)
195 case RTE_CRYPTO_CIPHER_3DES_CBC:
198 *algo = EVP_des_cbc();
201 *algo = EVP_des_ede_cbc();
204 *algo = EVP_des_ede3_cbc();
210 case RTE_CRYPTO_CIPHER_3DES_CTR:
212 case RTE_CRYPTO_CIPHER_AES_CBC:
215 *algo = EVP_aes_128_cbc();
218 *algo = EVP_aes_192_cbc();
221 *algo = EVP_aes_256_cbc();
227 case RTE_CRYPTO_CIPHER_AES_CTR:
230 *algo = EVP_aes_128_ctr();
233 *algo = EVP_aes_192_ctr();
236 *algo = EVP_aes_256_ctr();
253 /** Get adequate openssl function for input auth algorithm */
255 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
262 case RTE_CRYPTO_AUTH_MD5:
263 case RTE_CRYPTO_AUTH_MD5_HMAC:
266 case RTE_CRYPTO_AUTH_SHA1:
267 case RTE_CRYPTO_AUTH_SHA1_HMAC:
270 case RTE_CRYPTO_AUTH_SHA224:
271 case RTE_CRYPTO_AUTH_SHA224_HMAC:
272 *algo = EVP_sha224();
274 case RTE_CRYPTO_AUTH_SHA256:
275 case RTE_CRYPTO_AUTH_SHA256_HMAC:
276 *algo = EVP_sha256();
278 case RTE_CRYPTO_AUTH_SHA384:
279 case RTE_CRYPTO_AUTH_SHA384_HMAC:
280 *algo = EVP_sha384();
282 case RTE_CRYPTO_AUTH_SHA512:
283 case RTE_CRYPTO_AUTH_SHA512_HMAC:
284 *algo = EVP_sha512();
297 /** Get adequate openssl function for input cipher algorithm */
299 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
300 const EVP_CIPHER **algo)
306 case RTE_CRYPTO_AEAD_AES_GCM:
309 *algo = EVP_aes_128_gcm();
312 *algo = EVP_aes_192_gcm();
315 *algo = EVP_aes_256_gcm();
321 case RTE_CRYPTO_AEAD_AES_CCM:
324 *algo = EVP_aes_128_ccm();
327 *algo = EVP_aes_192_ccm();
330 *algo = EVP_aes_256_ccm();
347 /* Set session AEAD encryption parameters */
349 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
350 enum rte_crypto_aead_algorithm algo,
351 uint8_t tag_len, const uint8_t *key)
356 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
357 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
359 /* Select AEAD algo */
361 case RTE_CRYPTO_AEAD_AES_GCM:
362 iv_type = EVP_CTRL_GCM_SET_IVLEN;
367 case RTE_CRYPTO_AEAD_AES_CCM:
368 iv_type = EVP_CTRL_CCM_SET_IVLEN;
369 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
370 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
378 sess->cipher.mode = OPENSSL_CIPHER_LIB;
379 sess->cipher.ctx = EVP_CIPHER_CTX_new();
381 if (get_aead_algo(algo, sess->cipher.key.length,
382 &sess->cipher.evp_algo) != 0)
385 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
387 sess->chain_order = OPENSSL_CHAIN_COMBINED;
389 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
390 NULL, NULL, NULL) <= 0)
393 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
398 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
401 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
407 /* Set session AEAD decryption parameters */
409 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
410 enum rte_crypto_aead_algorithm algo,
411 uint8_t tag_len, const uint8_t *key)
414 unsigned int do_ccm = 0;
416 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
417 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
419 /* Select AEAD algo */
421 case RTE_CRYPTO_AEAD_AES_GCM:
422 iv_type = EVP_CTRL_GCM_SET_IVLEN;
426 case RTE_CRYPTO_AEAD_AES_CCM:
427 iv_type = EVP_CTRL_CCM_SET_IVLEN;
428 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
429 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
437 sess->cipher.mode = OPENSSL_CIPHER_LIB;
438 sess->cipher.ctx = EVP_CIPHER_CTX_new();
440 if (get_aead_algo(algo, sess->cipher.key.length,
441 &sess->cipher.evp_algo) != 0)
444 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
446 sess->chain_order = OPENSSL_CHAIN_COMBINED;
448 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
449 NULL, NULL, NULL) <= 0)
452 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
453 sess->iv.length, NULL) <= 0)
457 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
460 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
466 /** Set session cipher parameters */
468 openssl_set_session_cipher_parameters(struct openssl_session *sess,
469 const struct rte_crypto_sym_xform *xform)
471 /* Select cipher direction */
472 sess->cipher.direction = xform->cipher.op;
473 /* Select cipher key */
474 sess->cipher.key.length = xform->cipher.key.length;
476 /* Set IV parameters */
477 sess->iv.offset = xform->cipher.iv.offset;
478 sess->iv.length = xform->cipher.iv.length;
480 /* Select cipher algo */
481 switch (xform->cipher.algo) {
482 case RTE_CRYPTO_CIPHER_3DES_CBC:
483 case RTE_CRYPTO_CIPHER_AES_CBC:
484 case RTE_CRYPTO_CIPHER_AES_CTR:
485 sess->cipher.mode = OPENSSL_CIPHER_LIB;
486 sess->cipher.algo = xform->cipher.algo;
487 sess->cipher.ctx = EVP_CIPHER_CTX_new();
489 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
490 &sess->cipher.evp_algo) != 0)
493 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
494 sess->cipher.key.data);
495 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
496 if (EVP_EncryptInit_ex(sess->cipher.ctx,
497 sess->cipher.evp_algo,
498 NULL, xform->cipher.key.data,
502 } else if (sess->cipher.direction ==
503 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
504 if (EVP_DecryptInit_ex(sess->cipher.ctx,
505 sess->cipher.evp_algo,
506 NULL, xform->cipher.key.data,
514 case RTE_CRYPTO_CIPHER_3DES_CTR:
515 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
516 sess->cipher.ctx = EVP_CIPHER_CTX_new();
518 if (get_cipher_key_ede(xform->cipher.key.data,
519 sess->cipher.key.length,
520 sess->cipher.key.data) != 0)
524 case RTE_CRYPTO_CIPHER_DES_CBC:
525 sess->cipher.algo = xform->cipher.algo;
526 sess->cipher.ctx = EVP_CIPHER_CTX_new();
527 sess->cipher.evp_algo = EVP_des_cbc();
529 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
530 sess->cipher.key.data);
531 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
532 if (EVP_EncryptInit_ex(sess->cipher.ctx,
533 sess->cipher.evp_algo,
534 NULL, xform->cipher.key.data,
538 } else if (sess->cipher.direction ==
539 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
540 if (EVP_DecryptInit_ex(sess->cipher.ctx,
541 sess->cipher.evp_algo,
542 NULL, xform->cipher.key.data,
550 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
551 sess->cipher.algo = xform->cipher.algo;
552 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
553 sess->cipher.ctx = EVP_CIPHER_CTX_new();
554 sess->cipher.evp_algo = EVP_des_cbc();
556 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
557 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
558 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
559 NULL, xform->cipher.key.data, 0) != 1)
562 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
563 sess->cipher.key.data);
564 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
565 if (EVP_EncryptInit_ex(sess->cipher.ctx,
566 sess->cipher.evp_algo,
567 NULL, xform->cipher.key.data,
571 } else if (sess->cipher.direction ==
572 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
573 if (EVP_DecryptInit_ex(sess->cipher.ctx,
574 sess->cipher.evp_algo,
575 NULL, xform->cipher.key.data,
583 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
590 /* Set session auth parameters */
592 openssl_set_session_auth_parameters(struct openssl_session *sess,
593 const struct rte_crypto_sym_xform *xform)
595 /* Select auth generate/verify */
596 sess->auth.operation = xform->auth.op;
597 sess->auth.algo = xform->auth.algo;
599 sess->auth.digest_length = xform->auth.digest_length;
601 /* Select auth algo */
602 switch (xform->auth.algo) {
603 case RTE_CRYPTO_AUTH_AES_GMAC:
605 * OpenSSL requires GMAC to be a GCM operation
606 * with no cipher data length
608 sess->cipher.key.length = xform->auth.key.length;
610 /* Set IV parameters */
611 sess->iv.offset = xform->auth.iv.offset;
612 sess->iv.length = xform->auth.iv.length;
614 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
615 return openssl_set_sess_aead_enc_param(sess,
616 RTE_CRYPTO_AEAD_AES_GCM,
617 xform->auth.digest_length,
618 xform->auth.key.data);
620 return openssl_set_sess_aead_dec_param(sess,
621 RTE_CRYPTO_AEAD_AES_GCM,
622 xform->auth.digest_length,
623 xform->auth.key.data);
626 case RTE_CRYPTO_AUTH_MD5:
627 case RTE_CRYPTO_AUTH_SHA1:
628 case RTE_CRYPTO_AUTH_SHA224:
629 case RTE_CRYPTO_AUTH_SHA256:
630 case RTE_CRYPTO_AUTH_SHA384:
631 case RTE_CRYPTO_AUTH_SHA512:
632 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
633 if (get_auth_algo(xform->auth.algo,
634 &sess->auth.auth.evp_algo) != 0)
636 sess->auth.auth.ctx = EVP_MD_CTX_create();
639 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
640 case RTE_CRYPTO_AUTH_MD5_HMAC:
641 case RTE_CRYPTO_AUTH_SHA1_HMAC:
642 case RTE_CRYPTO_AUTH_SHA224_HMAC:
643 case RTE_CRYPTO_AUTH_SHA256_HMAC:
644 case RTE_CRYPTO_AUTH_SHA384_HMAC:
645 case RTE_CRYPTO_AUTH_SHA512_HMAC:
646 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
648 OSSL_PARAM params[2];
650 algo = digest_name_get(xform->auth.algo);
653 char algo_name[MAX_OSSL_ALGO_NAME_SIZE];
654 rte_memcpy(algo_name, algo, (sizeof(algo)+1));
656 EVP_MAC *mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
657 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac);
659 if (get_auth_algo(xform->auth.algo,
660 &sess->auth.hmac.evp_algo) != 0)
663 params[0] = OSSL_PARAM_construct_utf8_string("digest",
665 params[1] = OSSL_PARAM_construct_end();
666 if (EVP_MAC_init(sess->auth.hmac.ctx,
667 xform->auth.key.data,
668 xform->auth.key.length,
673 case RTE_CRYPTO_AUTH_MD5_HMAC:
674 case RTE_CRYPTO_AUTH_SHA1_HMAC:
675 case RTE_CRYPTO_AUTH_SHA224_HMAC:
676 case RTE_CRYPTO_AUTH_SHA256_HMAC:
677 case RTE_CRYPTO_AUTH_SHA384_HMAC:
678 case RTE_CRYPTO_AUTH_SHA512_HMAC:
679 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
680 sess->auth.hmac.ctx = HMAC_CTX_new();
681 if (get_auth_algo(xform->auth.algo,
682 &sess->auth.hmac.evp_algo) != 0)
685 if (HMAC_Init_ex(sess->auth.hmac.ctx,
686 xform->auth.key.data,
687 xform->auth.key.length,
688 sess->auth.hmac.evp_algo, NULL) != 1)
699 /* Set session AEAD parameters */
701 openssl_set_session_aead_parameters(struct openssl_session *sess,
702 const struct rte_crypto_sym_xform *xform)
704 /* Select cipher key */
705 sess->cipher.key.length = xform->aead.key.length;
707 /* Set IV parameters */
708 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
710 * For AES-CCM, the actual IV is placed
711 * one byte after the start of the IV field,
712 * according to the API.
714 sess->iv.offset = xform->aead.iv.offset + 1;
716 sess->iv.offset = xform->aead.iv.offset;
718 sess->iv.length = xform->aead.iv.length;
720 sess->auth.aad_length = xform->aead.aad_length;
721 sess->auth.digest_length = xform->aead.digest_length;
723 sess->aead_algo = xform->aead.algo;
724 /* Select cipher direction */
725 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
726 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
727 xform->aead.digest_length, xform->aead.key.data);
729 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
730 xform->aead.digest_length, xform->aead.key.data);
733 /** Parse crypto xform chain and set private session parameters */
735 openssl_set_session_parameters(struct openssl_session *sess,
736 const struct rte_crypto_sym_xform *xform)
738 const struct rte_crypto_sym_xform *cipher_xform = NULL;
739 const struct rte_crypto_sym_xform *auth_xform = NULL;
740 const struct rte_crypto_sym_xform *aead_xform = NULL;
743 sess->chain_order = openssl_get_chain_order(xform);
744 switch (sess->chain_order) {
745 case OPENSSL_CHAIN_ONLY_CIPHER:
746 cipher_xform = xform;
748 case OPENSSL_CHAIN_ONLY_AUTH:
751 case OPENSSL_CHAIN_CIPHER_AUTH:
752 cipher_xform = xform;
753 auth_xform = xform->next;
755 case OPENSSL_CHAIN_AUTH_CIPHER:
757 cipher_xform = xform->next;
759 case OPENSSL_CHAIN_COMBINED:
766 /* Default IV length = 0 */
769 /* cipher_xform must be check before auth_xform */
771 ret = openssl_set_session_cipher_parameters(
775 "Invalid/unsupported cipher parameters");
781 ret = openssl_set_session_auth_parameters(sess, auth_xform);
784 "Invalid/unsupported auth parameters");
790 ret = openssl_set_session_aead_parameters(sess, aead_xform);
793 "Invalid/unsupported AEAD parameters");
801 /** Reset private session parameters */
803 openssl_reset_session(struct openssl_session *sess)
805 EVP_CIPHER_CTX_free(sess->cipher.ctx);
807 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
808 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
810 switch (sess->auth.mode) {
811 case OPENSSL_AUTH_AS_AUTH:
812 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
814 case OPENSSL_AUTH_AS_HMAC:
815 EVP_PKEY_free(sess->auth.hmac.pkey);
816 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
817 EVP_MAC_CTX_free(sess->auth.hmac.ctx);
819 HMAC_CTX_free(sess->auth.hmac.ctx);
827 /** Provide session for operation */
829 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
831 struct openssl_session *sess = NULL;
832 struct openssl_asym_session *asym_sess = NULL;
834 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
835 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
836 /* get existing session */
837 if (likely(op->sym->session != NULL))
838 sess = (struct openssl_session *)
839 get_sym_session_private_data(
841 cryptodev_driver_id);
843 if (likely(op->asym->session != NULL))
844 asym_sess = (struct openssl_asym_session *)
845 op->asym->session->sess_private_data;
846 if (asym_sess == NULL)
848 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
852 /* sessionless asymmetric not supported */
853 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
856 /* provide internal session */
857 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
858 void *_sess_private_data = NULL;
863 if (rte_mempool_get(qp->sess_mp_priv,
864 (void **)&_sess_private_data))
867 sess = (struct openssl_session *)_sess_private_data;
869 if (unlikely(openssl_set_session_parameters(sess,
870 op->sym->xform) != 0)) {
871 rte_mempool_put(qp->sess_mp, _sess);
872 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
875 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
876 set_sym_session_private_data(op->sym->session,
877 cryptodev_driver_id, _sess_private_data);
881 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
887 *------------------------------------------------------------------------------
889 *------------------------------------------------------------------------------
892 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
893 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
898 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
900 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
902 offset -= rte_pktmbuf_data_len(m);
907 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
911 l = rte_pktmbuf_data_len(m) - offset;
913 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
919 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
925 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
926 uint8_t diff = l - dstlen, rem;
928 src = rte_pktmbuf_mtod(m, uint8_t *);
929 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
930 if (diff && inplace) {
932 (EVP_CIPHER_CTX_block_size(ctx) - diff));
933 if (EVP_EncryptUpdate(ctx, temp,
934 &dstlen, src, rem) <= 0)
937 rte_memcpy(*dst, temp, diff);
938 rte_memcpy(src, temp + diff, rem);
944 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
954 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
955 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
960 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
962 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
964 offset -= rte_pktmbuf_data_len(m);
969 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
973 l = rte_pktmbuf_data_len(m) - offset;
975 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
981 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
987 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
988 uint8_t diff = l - dstlen, rem;
990 src = rte_pktmbuf_mtod(m, uint8_t *);
991 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
992 if (diff && inplace) {
994 (EVP_CIPHER_CTX_block_size(ctx) - diff));
995 if (EVP_DecryptUpdate(ctx, temp,
996 &dstlen, src, rem) <= 0)
999 rte_memcpy(*dst, temp, diff);
1000 rte_memcpy(src, temp + diff, rem);
1006 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
1015 /** Process standard openssl cipher encryption */
1017 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1018 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1023 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1024 goto process_cipher_encrypt_err;
1026 EVP_CIPHER_CTX_set_padding(ctx, 0);
1028 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1029 srclen, ctx, inplace))
1030 goto process_cipher_encrypt_err;
1032 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
1033 goto process_cipher_encrypt_err;
1037 process_cipher_encrypt_err:
1038 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed");
1042 /** Process standard openssl cipher encryption */
1044 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
1045 uint8_t *iv, int srclen,
1046 EVP_CIPHER_CTX *ctx)
1049 uint8_t encrypted_iv[DES_BLOCK_SIZE];
1050 int encrypted_ivlen;
1052 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
1053 iv, DES_BLOCK_SIZE) <= 0)
1054 goto process_cipher_encrypt_err;
1056 for (i = 0; i < srclen; i++)
1057 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
1061 process_cipher_encrypt_err:
1062 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed");
1065 /** Process standard openssl cipher decryption */
1067 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
1068 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
1073 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1074 goto process_cipher_decrypt_err;
1076 EVP_CIPHER_CTX_set_padding(ctx, 0);
1078 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1079 srclen, ctx, inplace))
1080 goto process_cipher_decrypt_err;
1082 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
1083 goto process_cipher_decrypt_err;
1086 process_cipher_decrypt_err:
1087 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed");
1091 /** Process cipher des 3 ctr encryption, decryption algorithm */
1093 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
1094 int offset, uint8_t *iv, uint8_t *key, int srclen,
1095 EVP_CIPHER_CTX *ctx)
1097 uint8_t ebuf[8], ctr[8];
1103 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1105 offset -= rte_pktmbuf_data_len(m);
1108 goto process_cipher_des3ctr_err;
1110 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1111 l = rte_pktmbuf_data_len(m) - offset;
1113 /* We use 3DES encryption also for decryption.
1114 * IV is not important for 3DES ecb
1116 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
1117 goto process_cipher_des3ctr_err;
1121 for (n = 0; n < srclen; n++) {
1123 if (EVP_EncryptUpdate(ctx,
1124 (unsigned char *)&ebuf, &unused,
1125 (const unsigned char *)&ctr, 8) <= 0)
1126 goto process_cipher_des3ctr_err;
1129 dst[n] = *(src++) ^ ebuf[n % 8];
1135 src = rte_pktmbuf_mtod(m, uint8_t *);
1136 l = rte_pktmbuf_data_len(m);
1143 process_cipher_des3ctr_err:
1144 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed");
1148 /** Process AES-GCM encrypt algorithm */
1150 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1151 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1152 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1154 int len = 0, unused = 0;
1155 uint8_t empty[] = {};
1157 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1158 goto process_auth_encryption_gcm_err;
1161 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1162 goto process_auth_encryption_gcm_err;
1165 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1167 goto process_auth_encryption_gcm_err;
1169 /* Workaround open ssl bug in version less then 1.0.1f */
1170 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1171 goto process_auth_encryption_gcm_err;
1173 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1174 goto process_auth_encryption_gcm_err;
1176 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1177 goto process_auth_encryption_gcm_err;
1181 process_auth_encryption_gcm_err:
1182 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed");
1186 /** Process AES-CCM encrypt algorithm */
1188 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1189 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1190 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1194 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1195 goto process_auth_encryption_ccm_err;
1197 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1198 goto process_auth_encryption_ccm_err;
1202 * For AES-CCM, the actual AAD is placed
1203 * 18 bytes after the start of the AAD field,
1204 * according to the API.
1206 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1207 goto process_auth_encryption_ccm_err;
1210 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1212 goto process_auth_encryption_ccm_err;
1214 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1215 goto process_auth_encryption_ccm_err;
1217 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1218 goto process_auth_encryption_ccm_err;
1222 process_auth_encryption_ccm_err:
1223 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed");
1227 /** Process AES-GCM decrypt algorithm */
1229 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1230 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1231 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1233 int len = 0, unused = 0;
1234 uint8_t empty[] = {};
1236 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1237 goto process_auth_decryption_gcm_err;
1239 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1240 goto process_auth_decryption_gcm_err;
1243 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1244 goto process_auth_decryption_gcm_err;
1247 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1249 goto process_auth_decryption_gcm_err;
1251 /* Workaround open ssl bug in version less then 1.0.1f */
1252 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1253 goto process_auth_decryption_gcm_err;
1255 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1260 process_auth_decryption_gcm_err:
1261 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed");
1265 /** Process AES-CCM decrypt algorithm */
1267 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1268 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1269 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1270 EVP_CIPHER_CTX *ctx)
1274 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1275 goto process_auth_decryption_ccm_err;
1277 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1278 goto process_auth_decryption_ccm_err;
1280 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1281 goto process_auth_decryption_ccm_err;
1285 * For AES-CCM, the actual AAD is placed
1286 * 18 bytes after the start of the AAD field,
1287 * according to the API.
1289 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1290 goto process_auth_decryption_ccm_err;
1293 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1299 process_auth_decryption_ccm_err:
1300 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed");
1304 /** Process standard openssl auth algorithms */
1306 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1307 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1308 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1315 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1317 offset -= rte_pktmbuf_data_len(m);
1320 goto process_auth_err;
1322 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1323 goto process_auth_err;
1325 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1327 l = rte_pktmbuf_data_len(m) - offset;
1329 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1330 goto process_auth_err;
1331 goto process_auth_final;
1334 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1335 goto process_auth_err;
1339 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1340 src = rte_pktmbuf_mtod(m, uint8_t *);
1341 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1342 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1343 goto process_auth_err;
1348 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1349 goto process_auth_err;
1353 OPENSSL_LOG(ERR, "Process openssl auth failed");
1357 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1358 /** Process standard openssl auth algorithms with hmac */
1360 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1361 int srclen, EVP_MAC_CTX *ctx)
1368 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1370 offset -= rte_pktmbuf_data_len(m);
1373 goto process_auth_err;
1375 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1377 l = rte_pktmbuf_data_len(m) - offset;
1379 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1)
1380 goto process_auth_err;
1381 goto process_auth_final;
1384 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1385 goto process_auth_err;
1389 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1390 src = rte_pktmbuf_mtod(m, uint8_t *);
1391 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1392 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1)
1393 goto process_auth_err;
1398 if (EVP_MAC_final(ctx, dst, &dstlen, sizeof(dst)) != 1)
1399 goto process_auth_err;
1401 EVP_MAC_CTX_free(ctx);
1405 EVP_MAC_CTX_free(ctx);
1406 OPENSSL_LOG(ERR, "Process openssl auth failed");
1410 /** Process standard openssl auth algorithms with hmac */
1412 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1413 int srclen, HMAC_CTX *ctx)
1415 unsigned int dstlen;
1420 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1422 offset -= rte_pktmbuf_data_len(m);
1425 goto process_auth_err;
1427 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1429 l = rte_pktmbuf_data_len(m) - offset;
1431 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1432 goto process_auth_err;
1433 goto process_auth_final;
1436 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1437 goto process_auth_err;
1441 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1442 src = rte_pktmbuf_mtod(m, uint8_t *);
1443 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1444 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1445 goto process_auth_err;
1450 if (HMAC_Final(ctx, dst, &dstlen) != 1)
1451 goto process_auth_err;
1453 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1454 goto process_auth_err;
1459 OPENSSL_LOG(ERR, "Process openssl auth failed");
1463 /*----------------------------------------------------------------------------*/
1465 /** Process auth/cipher combined operation */
1467 process_openssl_combined_op
1468 (struct rte_crypto_op *op, struct openssl_session *sess,
1469 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1472 uint8_t *dst = NULL, *iv, *tag, *aad;
1473 int srclen, aadlen, status = -1;
1478 * Segmented destination buffer is not supported for
1479 * encryption/decryption
1481 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1482 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1486 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1488 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1490 offset = op->sym->auth.data.offset;
1491 aadlen = op->sym->auth.data.length;
1492 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1493 op->sym->auth.data.offset);
1494 tag = op->sym->auth.digest.data;
1496 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1499 srclen = op->sym->aead.data.length;
1500 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1501 op->sym->aead.data.offset);
1502 offset = op->sym->aead.data.offset;
1503 aad = op->sym->aead.aad.data;
1504 aadlen = sess->auth.aad_length;
1505 tag = op->sym->aead.digest.data;
1507 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1511 taglen = sess->auth.digest_length;
1513 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1514 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1515 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1516 status = process_openssl_auth_encryption_gcm(
1517 mbuf_src, offset, srclen,
1519 dst, tag, sess->cipher.ctx);
1521 status = process_openssl_auth_encryption_ccm(
1522 mbuf_src, offset, srclen,
1524 dst, tag, taglen, sess->cipher.ctx);
1527 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1528 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1529 status = process_openssl_auth_decryption_gcm(
1530 mbuf_src, offset, srclen,
1532 dst, tag, sess->cipher.ctx);
1534 status = process_openssl_auth_decryption_ccm(
1535 mbuf_src, offset, srclen,
1537 dst, tag, taglen, sess->cipher.ctx);
1541 if (status == (-EFAULT) &&
1542 sess->auth.operation ==
1543 RTE_CRYPTO_AUTH_OP_VERIFY)
1544 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1546 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1550 /** Process cipher operation */
1552 process_openssl_cipher_op
1553 (struct rte_crypto_op *op, struct openssl_session *sess,
1554 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1558 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0;
1559 EVP_CIPHER_CTX *ctx_copy;
1562 * Segmented OOP destination buffer is not supported for encryption/
1563 * decryption. In case of des3ctr, even inplace segmented buffers are
1566 if (!rte_pktmbuf_is_contiguous(mbuf_dst) &&
1567 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) {
1568 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1572 srclen = op->sym->cipher.data.length;
1573 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1574 op->sym->cipher.data.offset);
1576 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1578 ctx_copy = EVP_CIPHER_CTX_new();
1579 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx);
1581 if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1582 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1583 status = process_openssl_cipher_encrypt(mbuf_src, dst,
1584 op->sym->cipher.data.offset, iv,
1585 srclen, ctx_copy, inplace);
1587 status = process_openssl_cipher_decrypt(mbuf_src, dst,
1588 op->sym->cipher.data.offset, iv,
1589 srclen, ctx_copy, inplace);
1591 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1592 op->sym->cipher.data.offset, iv,
1593 sess->cipher.key.data, srclen,
1596 EVP_CIPHER_CTX_free(ctx_copy);
1598 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1601 /** Process cipher operation */
1603 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1604 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1605 struct rte_mbuf *mbuf_dst)
1607 uint8_t *src, *dst, *iv;
1608 uint8_t block_size, last_block_len;
1609 int srclen, status = 0;
1611 srclen = op->sym->cipher.data.length;
1612 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1613 op->sym->cipher.data.offset);
1614 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1615 op->sym->cipher.data.offset);
1617 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1620 block_size = DES_BLOCK_SIZE;
1622 last_block_len = srclen % block_size;
1623 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1624 /* Encrypt only with ECB mode XOR IV */
1625 if (srclen < block_size) {
1626 status = process_openssl_cipher_bpi_encrypt(src, dst,
1628 sess->cipher.bpi_ctx);
1630 srclen -= last_block_len;
1631 /* Encrypt with the block aligned stream with CBC mode */
1632 status = process_openssl_cipher_encrypt(mbuf_src, dst,
1633 op->sym->cipher.data.offset, iv,
1634 srclen, sess->cipher.ctx, 0);
1635 if (last_block_len) {
1636 /* Point at last block */
1639 * IV is the last encrypted block from
1640 * the previous operation
1642 iv = dst - block_size;
1644 srclen = last_block_len;
1645 /* Encrypt the last frame with ECB mode */
1646 status |= process_openssl_cipher_bpi_encrypt(src,
1648 srclen, sess->cipher.bpi_ctx);
1652 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1653 if (srclen < block_size) {
1654 status = process_openssl_cipher_bpi_encrypt(src, dst,
1657 sess->cipher.bpi_ctx);
1659 if (last_block_len) {
1660 /* Point at last block */
1661 dst += srclen - last_block_len;
1662 src += srclen - last_block_len;
1664 * IV is the last full block
1666 iv = src - block_size;
1668 * Decrypt the last frame with ECB mode
1669 * (encrypt, as it is the same operation)
1671 status = process_openssl_cipher_bpi_encrypt(src,
1673 last_block_len, sess->cipher.bpi_ctx);
1674 /* Prepare parameters for CBC mode op */
1675 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1677 dst += last_block_len - srclen;
1678 srclen -= last_block_len;
1681 /* Decrypt with CBC mode */
1682 status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1683 op->sym->cipher.data.offset, iv,
1684 srclen, sess->cipher.ctx, 0);
1689 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1692 /** Process auth operation */
1694 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1695 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1696 struct rte_mbuf *mbuf_dst)
1701 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1708 srclen = op->sym->auth.data.length;
1710 dst = qp->temp_digest;
1712 switch (sess->auth.mode) {
1713 case OPENSSL_AUTH_AS_AUTH:
1714 ctx_a = EVP_MD_CTX_create();
1715 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx);
1716 status = process_openssl_auth(mbuf_src, dst,
1717 op->sym->auth.data.offset, NULL, NULL, srclen,
1718 ctx_a, sess->auth.auth.evp_algo);
1719 EVP_MD_CTX_destroy(ctx_a);
1721 case OPENSSL_AUTH_AS_HMAC:
1722 # if OPENSSL_VERSION_NUMBER >= 0x30000000L
1723 mac = EVP_MAC_fetch(NULL, "HMAC", NULL);
1724 ctx_h = EVP_MAC_CTX_new(mac);
1725 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx);
1727 status = process_openssl_auth_hmac(mbuf_src, dst,
1728 op->sym->auth.data.offset, srclen,
1731 ctx_h = HMAC_CTX_new();
1732 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx);
1733 status = process_openssl_auth_hmac(mbuf_src, dst,
1734 op->sym->auth.data.offset, srclen,
1736 HMAC_CTX_free(ctx_h);
1744 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1745 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data,
1746 sess->auth.digest_length) != 0) {
1747 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1752 auth_dst = op->sym->auth.digest.data;
1753 if (auth_dst == NULL)
1754 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1755 op->sym->auth.data.offset +
1756 op->sym->auth.data.length);
1757 memcpy(auth_dst, dst, sess->auth.digest_length);
1761 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1764 /* process dsa sign operation */
1765 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
1767 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop,
1768 struct openssl_asym_session *sess)
1770 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1771 EVP_PKEY_CTX *dsa_ctx = NULL;
1772 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL);
1773 EVP_PKEY *pkey = NULL;
1774 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld;
1775 OSSL_PARAM *params = NULL;
1778 unsigned char *dsa_sign_data;
1779 const unsigned char *dsa_sign_data_p;
1781 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1782 params = OSSL_PARAM_BLD_to_param(param_bld);
1784 OSSL_PARAM_BLD_free(param_bld);
1789 || EVP_PKEY_fromdata_init(key_ctx) <= 0
1790 || EVP_PKEY_fromdata(key_ctx, &pkey,
1791 EVP_PKEY_PUBLIC_KEY, params) <= 0)
1794 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL);
1798 if (EVP_PKEY_sign_init(dsa_ctx) <= 0)
1801 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data,
1802 op->message.length) <= 0)
1808 dsa_sign_data = OPENSSL_malloc(outlen);
1812 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data,
1813 op->message.length) <= 0) {
1814 free(dsa_sign_data);
1818 dsa_sign_data_p = (const unsigned char *)dsa_sign_data;
1819 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen);
1821 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
1822 free(dsa_sign_data);
1825 const BIGNUM *r = NULL, *s = NULL;
1826 get_dsa_sign(sign, &r, &s);
1828 op->r.length = BN_bn2bin(r, op->r.data);
1829 op->s.length = BN_bn2bin(s, op->s.data);
1830 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1834 free(dsa_sign_data);
1839 OSSL_PARAM_free(params);
1841 EVP_PKEY_CTX_free(key_ctx);
1843 EVP_PKEY_CTX_free(dsa_ctx);
1847 /* process dsa verify operation */
1849 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop,
1850 struct openssl_asym_session *sess)
1852 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1853 DSA_SIG *sign = DSA_SIG_new();
1854 BIGNUM *r = NULL, *s = NULL;
1855 BIGNUM *pub_key = NULL;
1856 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld;
1857 OSSL_PARAM *params = NULL;
1858 EVP_PKEY *pkey = NULL;
1859 EVP_PKEY_CTX *dsa_ctx = NULL;
1860 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL);
1861 unsigned char *dsa_sig = NULL;
1865 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1867 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
1871 r = BN_bin2bn(op->r.data, op->r.length, r);
1872 s = BN_bin2bn(op->s.data, op->s.length, s);
1873 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key);
1874 if (!r || !s || !pub_key) {
1878 OSSL_PARAM_BLD_free(param_bld);
1879 goto err_dsa_verify;
1882 set_dsa_sign(sign, r, s);
1883 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) {
1884 OSSL_PARAM_BLD_free(param_bld);
1885 goto err_dsa_verify;
1888 params = OSSL_PARAM_BLD_to_param(param_bld);
1890 OSSL_PARAM_BLD_free(param_bld);
1891 goto err_dsa_verify;
1895 || EVP_PKEY_fromdata_init(key_ctx) <= 0
1896 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0)
1897 goto err_dsa_verify;
1899 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL);
1901 goto err_dsa_verify;
1904 goto err_dsa_verify;
1906 sig_len = i2d_DSA_SIG(sign, &dsa_sig);
1907 if (EVP_PKEY_verify_init(dsa_ctx) <= 0)
1908 goto err_dsa_verify;
1910 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len,
1911 op->message.data, op->message.length);
1913 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1921 OSSL_PARAM_free(params);
1923 EVP_PKEY_CTX_free(key_ctx);
1925 EVP_PKEY_CTX_free(dsa_ctx);
1931 process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
1932 struct openssl_asym_session *sess)
1934 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1935 DSA *dsa = sess->u.s.dsa;
1936 DSA_SIG *sign = NULL;
1938 sign = DSA_do_sign(op->message.data,
1943 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
1944 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1946 const BIGNUM *r = NULL, *s = NULL;
1947 get_dsa_sign(sign, &r, &s);
1949 op->r.length = BN_bn2bin(r, op->r.data);
1950 op->s.length = BN_bn2bin(s, op->s.data);
1951 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1959 /* process dsa verify operation */
1961 process_openssl_dsa_verify_op(struct rte_crypto_op *cop,
1962 struct openssl_asym_session *sess)
1964 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1965 DSA *dsa = sess->u.s.dsa;
1967 DSA_SIG *sign = DSA_SIG_new();
1968 BIGNUM *r = NULL, *s = NULL;
1969 BIGNUM *pub_key = NULL;
1972 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
1973 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1977 r = BN_bin2bn(op->r.data,
1980 s = BN_bin2bn(op->s.data,
1983 pub_key = BN_bin2bn(op->y.data,
1986 if (!r || !s || !pub_key) {
1991 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1994 set_dsa_sign(sign, r, s);
1995 set_dsa_pub_key(dsa, pub_key);
1997 ret = DSA_do_verify(op->message.data,
2003 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2005 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2013 /* process dh operation */
2014 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2016 process_openssl_dh_op_evp(struct rte_crypto_op *cop,
2017 struct openssl_asym_session *sess)
2019 struct rte_crypto_dh_op_param *op = &cop->asym->dh;
2020 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld;
2021 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer;
2022 OSSL_PARAM *params = NULL;
2023 EVP_PKEY *dhpkey = NULL;
2024 EVP_PKEY *peerkey = NULL;
2025 BIGNUM *priv_key = NULL;
2026 BIGNUM *pub_key = NULL;
2029 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2030 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
2031 if (dh_ctx == NULL || param_bld == NULL)
2034 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2035 OSSL_PARAM *params_peer = NULL;
2037 if (!param_bld_peer)
2040 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length,
2042 if (pub_key == NULL) {
2043 OSSL_PARAM_BLD_free(param_bld_peer);
2047 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY,
2049 OPENSSL_LOG(ERR, "Failed to set public key\n");
2050 OSSL_PARAM_BLD_free(param_bld_peer);
2055 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer);
2057 OSSL_PARAM_BLD_free(param_bld_peer);
2062 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
2063 if (EVP_PKEY_keygen_init(peer_ctx) != 1) {
2064 OSSL_PARAM_free(params_peer);
2069 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) {
2070 EVP_PKEY_CTX_free(peer_ctx);
2071 OSSL_PARAM_free(params_peer);
2076 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) {
2077 EVP_PKEY_CTX_free(peer_ctx);
2078 OSSL_PARAM_free(params_peer);
2083 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length,
2085 if (priv_key == NULL) {
2086 EVP_PKEY_CTX_free(peer_ctx);
2087 OSSL_PARAM_free(params_peer);
2092 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY,
2094 OPENSSL_LOG(ERR, "Failed to set private key\n");
2095 EVP_PKEY_CTX_free(peer_ctx);
2096 OSSL_PARAM_free(params_peer);
2102 OSSL_PARAM_free(params_peer);
2103 EVP_PKEY_CTX_free(peer_ctx);
2106 params = OSSL_PARAM_BLD_to_param(param_bld);
2110 if (EVP_PKEY_keygen_init(dh_ctx) != 1)
2113 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1)
2116 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1)
2119 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
2120 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__);
2121 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key))
2123 /* output public key */
2124 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data);
2127 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
2129 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__);
2130 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key))
2133 /* provide generated private key back to user */
2134 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data);
2137 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2139 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL);
2143 if (EVP_PKEY_derive_init(sc_ctx) <= 0) {
2144 EVP_PKEY_CTX_free(sc_ctx);
2149 EVP_PKEY_CTX_free(sc_ctx);
2153 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) {
2154 EVP_PKEY_CTX_free(sc_ctx);
2158 /* Determine buffer length */
2159 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) {
2160 EVP_PKEY_CTX_free(sc_ctx);
2164 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) {
2165 EVP_PKEY_CTX_free(sc_ctx);
2169 op->shared_secret.length = skey_len;
2170 EVP_PKEY_CTX_free(sc_ctx);
2173 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2182 OSSL_PARAM_free(params);
2184 EVP_PKEY_free(dhpkey);
2186 EVP_PKEY_free(peerkey);
2188 EVP_PKEY_CTX_free(dh_ctx);
2194 process_openssl_dh_op(struct rte_crypto_op *cop,
2195 struct openssl_asym_session *sess)
2197 struct rte_crypto_dh_op_param *op = &cop->asym->dh;
2198 struct rte_crypto_asym_op *asym_op = cop->asym;
2199 DH *dh_key = sess->u.dh.dh_key;
2200 BIGNUM *priv_key = NULL;
2203 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) {
2204 /* compute shared secret using peer public key
2205 * and current private key
2206 * shared secret = peer_key ^ priv_key mod p
2208 BIGNUM *peer_key = NULL;
2210 /* copy private key and peer key and compute shared secret */
2211 peer_key = BN_bin2bn(op->pub_key.data,
2214 if (peer_key == NULL) {
2215 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2218 priv_key = BN_bin2bn(op->priv_key.data,
2219 op->priv_key.length,
2221 if (priv_key == NULL) {
2223 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2226 ret = set_dh_priv_key(dh_key, priv_key);
2228 OPENSSL_LOG(ERR, "Failed to set private key\n");
2229 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2235 ret = DH_compute_key(
2236 op->shared_secret.data,
2239 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2241 /* priv key is already loaded into dh,
2242 * let's not free that directly here.
2243 * DH_free() will auto free it later.
2247 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2248 op->shared_secret.length = ret;
2254 * other options are public and private key generations.
2256 * if user provides private key,
2257 * then first set DH with user provided private key
2259 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE &&
2260 op->priv_key.length) {
2261 /* generate public key using user-provided private key
2262 * pub_key = g ^ priv_key mod p
2265 /* load private key into DH */
2266 priv_key = BN_bin2bn(op->priv_key.data,
2267 op->priv_key.length,
2269 if (priv_key == NULL) {
2270 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2273 ret = set_dh_priv_key(dh_key, priv_key);
2275 OPENSSL_LOG(ERR, "Failed to set private key\n");
2276 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2282 /* generate public and private key pair.
2284 * if private key already set, generates only public key.
2286 * if private key is not already set, then set it to random value
2287 * and update internal private key.
2289 if (!DH_generate_key(dh_key)) {
2290 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2294 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) {
2295 const BIGNUM *pub_key = NULL;
2297 OPENSSL_LOG(DEBUG, "%s:%d update public key\n",
2298 __func__, __LINE__);
2300 /* get the generated keys */
2301 get_dh_pub_key(dh_key, &pub_key);
2303 /* output public key */
2304 op->pub_key.length = BN_bn2bin(pub_key,
2308 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) {
2309 const BIGNUM *priv_key = NULL;
2311 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n",
2312 __func__, __LINE__);
2314 /* get the generated keys */
2315 get_dh_priv_key(dh_key, &priv_key);
2317 /* provide generated private key back to user */
2318 op->priv_key.length = BN_bn2bin(priv_key,
2322 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2328 /* process modinv operation */
2330 process_openssl_modinv_op(struct rte_crypto_op *cop,
2331 struct openssl_asym_session *sess)
2333 struct rte_crypto_asym_op *op = cop->asym;
2334 BIGNUM *base = BN_CTX_get(sess->u.m.ctx);
2335 BIGNUM *res = BN_CTX_get(sess->u.m.ctx);
2337 if (unlikely(base == NULL || res == NULL)) {
2340 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2344 base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
2345 op->modinv.base.length, base);
2347 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) {
2348 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2349 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data);
2351 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2360 /* process modexp operation */
2362 process_openssl_modexp_op(struct rte_crypto_op *cop,
2363 struct openssl_asym_session *sess)
2365 struct rte_crypto_asym_op *op = cop->asym;
2366 BIGNUM *base = BN_CTX_get(sess->u.e.ctx);
2367 BIGNUM *res = BN_CTX_get(sess->u.e.ctx);
2369 if (unlikely(base == NULL || res == NULL)) {
2372 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2376 base = BN_bin2bn((const unsigned char *)op->modex.base.data,
2377 op->modex.base.length, base);
2379 if (BN_mod_exp(res, base, sess->u.e.exp,
2380 sess->u.e.mod, sess->u.e.ctx)) {
2381 op->modex.result.length = BN_bn2bin(res, op->modex.result.data);
2382 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2384 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2393 /* process rsa operations */
2394 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2396 process_openssl_rsa_op_evp(struct rte_crypto_op *cop,
2397 struct openssl_asym_session *sess)
2399 struct rte_crypto_asym_op *op = cop->asym;
2400 uint32_t pad = (op->rsa.padding.type);
2405 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2406 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx;
2411 case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
2412 pad = RSA_PKCS1_PADDING;
2414 case RTE_CRYPTO_RSA_PADDING_NONE:
2415 pad = RSA_NO_PADDING;
2418 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2420 "rsa pad type not supported %d\n", pad);
2424 switch (op->rsa.op_type) {
2425 case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2426 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1)
2429 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2432 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen,
2433 op->rsa.message.data,
2434 op->rsa.message.length) <= 0)
2440 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen,
2441 op->rsa.message.data,
2442 op->rsa.message.length) <= 0)
2444 op->rsa.cipher.length = outlen;
2447 "length of encrypted text %zu\n", outlen);
2450 case RTE_CRYPTO_ASYM_OP_DECRYPT:
2451 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1)
2454 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2457 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen,
2458 op->rsa.cipher.data,
2459 op->rsa.cipher.length) <= 0)
2465 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen,
2466 op->rsa.cipher.data,
2467 op->rsa.cipher.length) <= 0)
2469 op->rsa.message.length = outlen;
2471 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen);
2474 case RTE_CRYPTO_ASYM_OP_SIGN:
2475 if (EVP_PKEY_sign_init(rsa_ctx) <= 0)
2478 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0)
2481 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen,
2482 op->rsa.message.data,
2483 op->rsa.message.length) <= 0)
2485 op->rsa.sign.length = outlen;
2488 case RTE_CRYPTO_ASYM_OP_VERIFY:
2489 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
2491 OPENSSL_LOG(ERR, "Memory allocation failed");
2495 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) {
2500 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) {
2505 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen,
2507 op->rsa.sign.length) <= 0) {
2513 "Length of public_decrypt %zu "
2514 "length of message %zd\n",
2515 outlen, op->rsa.message.length);
2516 if (CRYPTO_memcmp(tmp, op->rsa.message.data,
2517 op->rsa.message.length)) {
2518 OPENSSL_LOG(ERR, "RSA sign Verification failed");
2524 /* allow ops with invalid args to be pushed to
2527 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2532 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2539 process_openssl_rsa_op(struct rte_crypto_op *cop,
2540 struct openssl_asym_session *sess)
2543 struct rte_crypto_asym_op *op = cop->asym;
2544 RSA *rsa = sess->u.r.rsa;
2545 uint32_t pad = (op->rsa.padding.type);
2548 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2551 case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
2552 pad = RSA_PKCS1_PADDING;
2554 case RTE_CRYPTO_RSA_PADDING_NONE:
2555 pad = RSA_NO_PADDING;
2558 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2560 "rsa pad type not supported %d\n", pad);
2564 switch (op->rsa.op_type) {
2565 case RTE_CRYPTO_ASYM_OP_ENCRYPT:
2566 ret = RSA_public_encrypt(op->rsa.message.length,
2567 op->rsa.message.data,
2568 op->rsa.cipher.data,
2573 op->rsa.cipher.length = ret;
2575 "length of encrypted text %d\n", ret);
2578 case RTE_CRYPTO_ASYM_OP_DECRYPT:
2579 ret = RSA_private_decrypt(op->rsa.cipher.length,
2580 op->rsa.cipher.data,
2581 op->rsa.message.data,
2585 op->rsa.message.length = ret;
2588 case RTE_CRYPTO_ASYM_OP_SIGN:
2589 ret = RSA_private_encrypt(op->rsa.message.length,
2590 op->rsa.message.data,
2595 op->rsa.sign.length = ret;
2598 case RTE_CRYPTO_ASYM_OP_VERIFY:
2599 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
2601 OPENSSL_LOG(ERR, "Memory allocation failed");
2602 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2605 ret = RSA_public_decrypt(op->rsa.sign.length,
2612 "Length of public_decrypt %d "
2613 "length of message %zd\n",
2614 ret, op->rsa.message.length);
2615 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data,
2616 op->rsa.message.length))) {
2617 OPENSSL_LOG(ERR, "RSA sign Verification failed");
2618 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2624 /* allow ops with invalid args to be pushed to
2627 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2632 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
2639 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2640 struct openssl_asym_session *sess)
2644 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2646 switch (sess->xfrm_type) {
2647 case RTE_CRYPTO_ASYM_XFORM_RSA:
2648 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2649 retval = process_openssl_rsa_op_evp(op, sess);
2651 retval = process_openssl_rsa_op(op, sess);
2654 case RTE_CRYPTO_ASYM_XFORM_MODEX:
2655 retval = process_openssl_modexp_op(op, sess);
2657 case RTE_CRYPTO_ASYM_XFORM_MODINV:
2658 retval = process_openssl_modinv_op(op, sess);
2660 case RTE_CRYPTO_ASYM_XFORM_DH:
2661 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2662 retval = process_openssl_dh_op_evp(op, sess);
2664 retval = process_openssl_dh_op(op, sess);
2667 case RTE_CRYPTO_ASYM_XFORM_DSA:
2668 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2669 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
2670 retval = process_openssl_dsa_sign_op_evp(op, sess);
2671 else if (op->asym->dsa.op_type ==
2672 RTE_CRYPTO_ASYM_OP_VERIFY)
2674 process_openssl_dsa_verify_op_evp(op, sess);
2676 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
2677 retval = process_openssl_dsa_sign_op(op, sess);
2678 else if (op->asym->dsa.op_type ==
2679 RTE_CRYPTO_ASYM_OP_VERIFY)
2681 process_openssl_dsa_verify_op(op, sess);
2683 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2687 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2691 /* op processed so push to completion queue as processed */
2692 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2694 /* return error if failed to put in completion queue */
2702 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst,
2703 struct rte_crypto_op *op)
2705 uint8_t *p_src, *p_dst;
2707 p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
2708 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
2711 * Copy the content between cipher offset and auth offset
2712 * for generating correct digest.
2714 if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
2715 memcpy(p_dst + op->sym->auth.data.offset,
2716 p_src + op->sym->auth.data.offset,
2717 op->sym->cipher.data.offset -
2718 op->sym->auth.data.offset);
2721 /** Process crypto operation for mbuf */
2723 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2724 struct openssl_session *sess)
2726 struct rte_mbuf *msrc, *mdst;
2729 msrc = op->sym->m_src;
2730 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
2732 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2734 switch (sess->chain_order) {
2735 case OPENSSL_CHAIN_ONLY_CIPHER:
2736 process_openssl_cipher_op(op, sess, msrc, mdst);
2738 case OPENSSL_CHAIN_ONLY_AUTH:
2739 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2741 case OPENSSL_CHAIN_CIPHER_AUTH:
2742 process_openssl_cipher_op(op, sess, msrc, mdst);
2745 copy_plaintext(msrc, mdst, op);
2746 process_openssl_auth_op(qp, op, sess, mdst, mdst);
2748 case OPENSSL_CHAIN_AUTH_CIPHER:
2749 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2750 process_openssl_cipher_op(op, sess, msrc, mdst);
2752 case OPENSSL_CHAIN_COMBINED:
2753 process_openssl_combined_op(op, sess, msrc, mdst);
2755 case OPENSSL_CHAIN_CIPHER_BPI:
2756 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
2759 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
2763 /* Free session if a session-less crypto op */
2764 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
2765 openssl_reset_session(sess);
2766 memset(sess, 0, sizeof(struct openssl_session));
2767 memset(op->sym->session, 0,
2768 rte_cryptodev_sym_get_existing_header_session_size(
2770 rte_mempool_put(qp->sess_mp_priv, sess);
2771 rte_mempool_put(qp->sess_mp, op->sym->session);
2772 op->sym->session = NULL;
2775 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
2776 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2778 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
2779 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2787 *------------------------------------------------------------------------------
2789 *------------------------------------------------------------------------------
2792 /** Enqueue burst */
2794 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
2798 struct openssl_qp *qp = queue_pair;
2801 for (i = 0; i < nb_ops; i++) {
2802 sess = get_session(qp, ops[i]);
2803 if (unlikely(sess == NULL))
2806 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
2807 retval = process_op(qp, ops[i],
2808 (struct openssl_session *) sess);
2810 retval = process_asym_op(qp, ops[i],
2811 (struct openssl_asym_session *) sess);
2812 if (unlikely(retval < 0))
2816 qp->stats.enqueued_count += i;
2820 qp->stats.enqueue_err_count++;
2824 /** Dequeue burst */
2826 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
2829 struct openssl_qp *qp = queue_pair;
2831 unsigned int nb_dequeued = 0;
2833 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
2834 (void **)ops, nb_ops, NULL);
2835 qp->stats.dequeued_count += nb_dequeued;
2840 /** Create OPENSSL crypto device */
2842 cryptodev_openssl_create(const char *name,
2843 struct rte_vdev_device *vdev,
2844 struct rte_cryptodev_pmd_init_params *init_params)
2846 struct rte_cryptodev *dev;
2847 struct openssl_private *internals;
2849 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2851 OPENSSL_LOG(ERR, "failed to create cryptodev vdev");
2855 dev->driver_id = cryptodev_driver_id;
2856 dev->dev_ops = rte_openssl_pmd_ops;
2858 /* register rx/tx burst functions for data path */
2859 dev->dequeue_burst = openssl_pmd_dequeue_burst;
2860 dev->enqueue_burst = openssl_pmd_enqueue_burst;
2862 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2863 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2864 RTE_CRYPTODEV_FF_CPU_AESNI |
2865 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2866 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2867 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2868 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
2869 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP |
2870 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT |
2871 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
2873 internals = dev->data->dev_private;
2875 internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
2877 rte_cryptodev_pmd_probing_finish(dev);
2879 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2880 /* Load legacy provider
2881 * Some algorithms are no longer available in earlier version of openssl,
2882 * unless the legacy provider explicitly loaded. e.g. DES
2884 ossl_legacy_provider_load();
2889 OPENSSL_LOG(ERR, "driver %s: create failed",
2892 cryptodev_openssl_remove(vdev);
2896 /** Initialise OPENSSL crypto device */
2898 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
2900 struct rte_cryptodev_pmd_init_params init_params = {
2902 sizeof(struct openssl_private),
2904 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2907 const char *input_args;
2909 name = rte_vdev_device_name(vdev);
2912 input_args = rte_vdev_device_args(vdev);
2914 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2916 return cryptodev_openssl_create(name, vdev, &init_params);
2919 /** Uninitialise OPENSSL crypto device */
2921 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
2923 struct rte_cryptodev *cryptodev;
2926 name = rte_vdev_device_name(vdev);
2930 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2931 if (cryptodev == NULL)
2934 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
2935 ossl_legacy_provider_unload();
2937 return rte_cryptodev_pmd_destroy(cryptodev);
2940 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
2941 .probe = cryptodev_openssl_probe,
2942 .remove = cryptodev_openssl_remove
2945 static struct cryptodev_driver openssl_crypto_drv;
2947 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
2948 cryptodev_openssl_pmd_drv);
2949 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
2950 "max_nb_queue_pairs=<int> "
2952 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
2953 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);
2954 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO);