X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fopenssl%2Frte_openssl_pmd.c;h=5d29171dd4df7a4a3a4227f4a5266b975444e10e;hb=b8d08b0dc34dec99343d183e0cbdba456d54bfd0;hp=312154a561dc29be32d141e043e8fd50ce72a558;hpb=5d63ef31d8a51bb318d10c4d0ce99cc4184a6d70;p=dpdk.git diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index 312154a561..5d29171dd4 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -42,31 +43,11 @@ #include "rte_openssl_pmd_private.h" -static int cryptodev_openssl_remove(const char *name); +#define DES_BLOCK_SIZE 8 -/*----------------------------------------------------------------------------*/ - -/** - * Global static parameter used to create a unique name for each - * OPENSSL crypto device. - */ -static unsigned int unique_name_id; - -static inline int -create_unique_device_name(char *name, size_t size) -{ - int ret; - - if (name == NULL) - return -EINVAL; +static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); - ret = snprintf(name, size, "%s_%u", - RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), - unique_name_id++); - if (ret < 0) - return ret; - return 0; -} +/*----------------------------------------------------------------------------*/ /** * Increment counter by 1 @@ -311,7 +292,21 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, sess->cipher.key.data) != 0) return -EINVAL; break; + case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: + sess->cipher.algo = xform->cipher.algo; + sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + sess->cipher.evp_algo = EVP_des_cbc(); + sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); + /* IV will be ECB encrypted whether direction is encrypt or decrypt */ + if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), + NULL, xform->cipher.key.data, 0) != 1) + return -EINVAL; + + get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, + sess->cipher.key.data); + break; default: sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; return -EINVAL; @@ -429,6 +424,9 @@ openssl_reset_session(struct openssl_session *sess) { EVP_CIPHER_CTX_free(sess->cipher.ctx); + if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) + EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); + switch (sess->auth.mode) { case OPENSSL_AUTH_AS_AUTH: EVP_MD_CTX_destroy(sess->auth.auth.ctx); @@ -484,24 +482,112 @@ get_session(struct openssl_qp *qp, struct rte_crypto_op *op) * Process Operations *------------------------------------------------------------------------------ */ +static inline int +process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, + uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) +{ + struct rte_mbuf *m; + int dstlen; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + return -1; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) + return -1; + *dst += l; + return 0; + } + + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + + *dst += dstlen; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + *dst += dstlen; + n -= l; + } + + return 0; +} + +static inline int +process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, + uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) +{ + struct rte_mbuf *m; + int dstlen; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + return -1; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) + return -1; + *dst += l; + return 0; + } + + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + + *dst += dstlen; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + *dst += dstlen; + n -= l; + } + + return 0; +} /** Process standard openssl cipher encryption */ static int -process_openssl_cipher_encrypt(uint8_t *src, uint8_t *dst, - uint8_t *iv, uint8_t *key, int srclen, +process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, uint8_t *key, int srclen, EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) { - int dstlen, totlen; + int totlen; if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0) goto process_cipher_encrypt_err; EVP_CIPHER_CTX_set_padding(ctx, 0); - if (EVP_EncryptUpdate(ctx, dst, &dstlen, src, srclen) <= 0) + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_cipher_encrypt_err; - if (EVP_EncryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0) + if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) goto process_cipher_encrypt_err; return 0; @@ -511,25 +597,48 @@ process_cipher_encrypt_err: return -EINVAL; } +/** Process standard openssl cipher encryption */ +static int +process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, + uint8_t *iv, int srclen, + EVP_CIPHER_CTX *ctx) +{ + uint8_t i; + uint8_t encrypted_iv[DES_BLOCK_SIZE]; + int encrypted_ivlen; + + if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, + iv, DES_BLOCK_SIZE) <= 0) + goto process_cipher_encrypt_err; + + for (i = 0; i < srclen; i++) + *(dst + i) = *(src + i) ^ (encrypted_iv[i]); + + return 0; + +process_cipher_encrypt_err: + OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed"); + return -EINVAL; +} /** Process standard openssl cipher decryption */ static int -process_openssl_cipher_decrypt(uint8_t *src, uint8_t *dst, - uint8_t *iv, uint8_t *key, int srclen, +process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, uint8_t *key, int srclen, EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) { - int dstlen, totlen; + int totlen; if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0) goto process_cipher_decrypt_err; EVP_CIPHER_CTX_set_padding(ctx, 0); - if (EVP_DecryptUpdate(ctx, dst, &dstlen, src, srclen) <= 0) + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_cipher_decrypt_err; - if (EVP_DecryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0) + if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) goto process_cipher_decrypt_err; - return 0; process_cipher_decrypt_err: @@ -539,11 +648,25 @@ process_cipher_decrypt_err: /** Process cipher des 3 ctr encryption, decryption algorithm */ static int -process_openssl_cipher_des3ctr(uint8_t *src, uint8_t *dst, - uint8_t *iv, uint8_t *key, int srclen, EVP_CIPHER_CTX *ctx) +process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, uint8_t *key, int srclen, + EVP_CIPHER_CTX *ctx) { uint8_t ebuf[8], ctr[8]; int unused, n; + struct rte_mbuf *m; + uint8_t *src; + int l; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_cipher_des3ctr_err; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + l = rte_pktmbuf_data_len(m) - offset; /* We use 3DES encryption also for decryption. * IV is not important for 3DES ecb @@ -552,9 +675,8 @@ process_openssl_cipher_des3ctr(uint8_t *src, uint8_t *dst, goto process_cipher_des3ctr_err; memcpy(ctr, iv, 8); - n = 0; - while (n < srclen) { + for (n = 0; n < srclen; n++) { if (n % 8 == 0) { if (EVP_EncryptUpdate(ctx, (unsigned char *)&ebuf, &unused, @@ -562,8 +684,16 @@ process_openssl_cipher_des3ctr(uint8_t *src, uint8_t *dst, goto process_cipher_des3ctr_err; ctr_inc(ctr); } - dst[n] = src[n] ^ ebuf[n % 8]; - n++; + dst[n] = *(src++) ^ ebuf[n % 8]; + + l--; + if (!l) { + m = m->next; + if (m) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m); + } + } } return 0; @@ -575,9 +705,9 @@ process_cipher_des3ctr_err: /** Process auth/encription aes-gcm algorithm */ static int -process_openssl_auth_encryption_gcm(uint8_t *src, int srclen, - uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, - uint8_t *key, uint8_t *dst, uint8_t *tag, +process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, + uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) { int len = 0, unused = 0; @@ -592,20 +722,20 @@ process_openssl_auth_encryption_gcm(uint8_t *src, int srclen, if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) goto process_auth_encryption_gcm_err; - if (aadlen > 0) { + if (aadlen > 0) if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) goto process_auth_encryption_gcm_err; - /* Workaround open ssl bug in version less then 1.0.1f */ - if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) - goto process_auth_encryption_gcm_err; - } - if (srclen > 0) - if (EVP_EncryptUpdate(ctx, dst, &len, src, srclen) <= 0) + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_auth_encryption_gcm_err; - if (EVP_EncryptFinal_ex(ctx, dst + len, &len) <= 0) + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_encryption_gcm_err; + + if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) goto process_auth_encryption_gcm_err; if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) @@ -619,10 +749,10 @@ process_auth_encryption_gcm_err: } static int -process_openssl_auth_decryption_gcm(uint8_t *src, int srclen, - uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, - uint8_t *key, uint8_t *dst, uint8_t *tag, - EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) +process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, + uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, + const EVP_CIPHER *algo) { int len = 0, unused = 0; uint8_t empty[] = {}; @@ -639,20 +769,20 @@ process_openssl_auth_decryption_gcm(uint8_t *src, int srclen, if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) goto process_auth_decryption_gcm_err; - if (aadlen > 0) { + if (aadlen > 0) if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) goto process_auth_decryption_gcm_err; - /* Workaround open ssl bug in version less then 1.0.1f */ - if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) - goto process_auth_decryption_gcm_err; - } - if (srclen > 0) - if (EVP_DecryptUpdate(ctx, dst, &len, src, srclen) <= 0) + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) goto process_auth_decryption_gcm_err; - if (EVP_DecryptFinal_ex(ctx, dst + len, &len) <= 0) + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_decryption_gcm_err; + + if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) goto process_auth_decryption_gcm_final_err; return 0; @@ -667,21 +797,50 @@ process_auth_decryption_gcm_final_err: /** Process standard openssl auth algorithms */ static int -process_openssl_auth(uint8_t *src, uint8_t *dst, +process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) { size_t dstlen; + struct rte_mbuf *m; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_auth_err; if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) goto process_auth_err; - if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) + goto process_auth_err; + goto process_auth_final; + } + + if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) goto process_auth_err; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) + goto process_auth_err; + n -= l; + } + +process_auth_final: if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) goto process_auth_err; - return 0; process_auth_err: @@ -691,18 +850,48 @@ process_auth_err: /** Process standard openssl auth algorithms with hmac */ static int -process_openssl_auth_hmac(uint8_t *src, uint8_t *dst, +process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, __rte_unused uint8_t *iv, EVP_PKEY *pkey, - int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) + int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) { size_t dstlen; + struct rte_mbuf *m; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_auth_err; if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0) goto process_auth_err; - if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) + goto process_auth_err; + goto process_auth_final; + } + + if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) goto process_auth_err; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) + goto process_auth_err; + n -= l; + } + +process_auth_final: if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0) goto process_auth_err; @@ -722,9 +911,18 @@ process_openssl_combined_op struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) { /* cipher */ - uint8_t *src = NULL, *dst = NULL, *iv, *tag, *aad; + uint8_t *dst = NULL, *iv, *tag, *aad; int srclen, ivlen, aadlen, status = -1; + /* + * Segmented destination buffer is not supported for + * encryption/decryption + */ + if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return; + } + iv = op->sym->cipher.iv.data; ivlen = op->sym->cipher.iv.length; aad = op->sym->auth.aad.data; @@ -740,22 +938,22 @@ process_openssl_combined_op srclen = 0; else { srclen = op->sym->cipher.data.length; - src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, - op->sym->cipher.data.offset); dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); } if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) status = process_openssl_auth_encryption_gcm( - src, srclen, aad, aadlen, iv, ivlen, - sess->cipher.key.data, dst, tag, - sess->cipher.ctx, sess->cipher.evp_algo); + mbuf_src, op->sym->cipher.data.offset, srclen, + aad, aadlen, iv, ivlen, sess->cipher.key.data, + dst, tag, sess->cipher.ctx, + sess->cipher.evp_algo); else status = process_openssl_auth_decryption_gcm( - src, srclen, aad, aadlen, iv, ivlen, - sess->cipher.key.data, dst, tag, - sess->cipher.ctx, sess->cipher.evp_algo); + mbuf_src, op->sym->cipher.data.offset, srclen, + aad, aadlen, iv, ivlen, sess->cipher.key.data, + dst, tag, sess->cipher.ctx, + sess->cipher.evp_algo); if (status != 0) { if (status == (-EFAULT) && @@ -773,12 +971,19 @@ process_openssl_cipher_op (struct rte_crypto_op *op, struct openssl_session *sess, struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) { - uint8_t *src, *dst, *iv; + uint8_t *dst, *iv; int srclen, status; + /* + * Segmented destination buffer is not supported for + * encryption/decryption + */ + if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return; + } + srclen = op->sym->cipher.data.length; - src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, - op->sym->cipher.data.offset); dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); @@ -786,17 +991,20 @@ process_openssl_cipher_op if (sess->cipher.mode == OPENSSL_CIPHER_LIB) if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) - status = process_openssl_cipher_encrypt(src, dst, iv, + status = process_openssl_cipher_encrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, sess->cipher.key.data, srclen, sess->cipher.ctx, sess->cipher.evp_algo); else - status = process_openssl_cipher_decrypt(src, dst, iv, + status = process_openssl_cipher_decrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, sess->cipher.key.data, srclen, sess->cipher.ctx, sess->cipher.evp_algo); else - status = process_openssl_cipher_des3ctr(src, dst, iv, + status = process_openssl_cipher_des3ctr(mbuf_src, dst, + op->sym->cipher.data.offset, iv, sess->cipher.key.data, srclen, sess->cipher.ctx); @@ -804,18 +1012,108 @@ process_openssl_cipher_op op->status = RTE_CRYPTO_OP_STATUS_ERROR; } +/** Process cipher operation */ +static void +process_openssl_docsis_bpi_op(struct rte_crypto_op *op, + struct openssl_session *sess, struct rte_mbuf *mbuf_src, + struct rte_mbuf *mbuf_dst) +{ + uint8_t *src, *dst, *iv; + uint8_t block_size, last_block_len; + int srclen, status = 0; + + srclen = op->sym->cipher.data.length; + src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, + op->sym->cipher.data.offset); + dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + op->sym->cipher.data.offset); + + iv = op->sym->cipher.iv.data; + + block_size = DES_BLOCK_SIZE; + + last_block_len = srclen % block_size; + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + /* Encrypt only with ECB mode XOR IV */ + if (srclen < block_size) { + status = process_openssl_cipher_bpi_encrypt(src, dst, + iv, srclen, + sess->cipher.bpi_ctx); + } else { + srclen -= last_block_len; + /* Encrypt with the block aligned stream with CBC mode */ + status = process_openssl_cipher_encrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, + sess->cipher.key.data, srclen, + sess->cipher.ctx, sess->cipher.evp_algo); + if (last_block_len) { + /* Point at last block */ + dst += srclen; + /* + * IV is the last encrypted block from + * the previous operation + */ + iv = dst - block_size; + src += srclen; + srclen = last_block_len; + /* Encrypt the last frame with ECB mode */ + status |= process_openssl_cipher_bpi_encrypt(src, + dst, iv, + srclen, sess->cipher.bpi_ctx); + } + } + } else { + /* Decrypt only with ECB mode (encrypt, as it is same operation) */ + if (srclen < block_size) { + status = process_openssl_cipher_bpi_encrypt(src, dst, + iv, + srclen, + sess->cipher.bpi_ctx); + } else { + if (last_block_len) { + /* Point at last block */ + dst += srclen - last_block_len; + src += srclen - last_block_len; + /* + * IV is the last full block + */ + iv = src - block_size; + /* + * Decrypt the last frame with ECB mode + * (encrypt, as it is the same operation) + */ + status = process_openssl_cipher_bpi_encrypt(src, + dst, iv, + last_block_len, sess->cipher.bpi_ctx); + /* Prepare parameters for CBC mode op */ + iv = op->sym->cipher.iv.data; + dst += last_block_len - srclen; + srclen -= last_block_len; + } + + /* Decrypt with CBC mode */ + status |= process_openssl_cipher_decrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, + sess->cipher.key.data, srclen, + sess->cipher.ctx, + sess->cipher.evp_algo); + } + } + + if (status != 0) + op->status = RTE_CRYPTO_OP_STATUS_ERROR; +} + /** Process auth operation */ static void process_openssl_auth_op (struct rte_crypto_op *op, struct openssl_session *sess, struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) { - uint8_t *src, *dst; + uint8_t *dst; int srclen, status; srclen = op->sym->auth.data.length; - src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, - op->sym->auth.data.offset); if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, @@ -830,13 +1128,14 @@ process_openssl_auth_op switch (sess->auth.mode) { case OPENSSL_AUTH_AS_AUTH: - status = process_openssl_auth(src, dst, - NULL, NULL, srclen, + status = process_openssl_auth(mbuf_src, dst, + op->sym->auth.data.offset, NULL, NULL, srclen, sess->auth.auth.ctx, sess->auth.auth.evp_algo); break; case OPENSSL_AUTH_AS_HMAC: - status = process_openssl_auth_hmac(src, dst, - NULL, sess->auth.hmac.pkey, srclen, + status = process_openssl_auth_hmac(mbuf_src, dst, + op->sym->auth.data.offset, NULL, + sess->auth.hmac.pkey, srclen, sess->auth.hmac.ctx, sess->auth.hmac.evp_algo); break; default: @@ -850,8 +1149,7 @@ process_openssl_auth_op op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(mbuf_src, - op->sym->auth.digest.length); + rte_pktmbuf_trim(mbuf_src, op->sym->auth.digest.length); } if (status != 0) @@ -889,6 +1187,9 @@ process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, case OPENSSL_CHAIN_COMBINED: process_openssl_combined_op(op, sess, msrc, mdst); break; + case OPENSSL_CHAIN_CIPHER_BPI: + process_openssl_docsis_bpi_op(op, sess, msrc, mdst); + break; default: op->status = RTE_CRYPTO_OP_STATUS_ERROR; break; @@ -902,7 +1203,6 @@ process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, op->sym->session = NULL; } - if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -957,7 +1257,7 @@ openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, unsigned int nb_dequeued = 0; nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, - (void **)ops, nb_ops); + (void **)ops, nb_ops, NULL); qp->stats.dequeued_count += nb_dequeued; return nb_dequeued; @@ -966,22 +1266,20 @@ openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, /** Create OPENSSL crypto device */ static int cryptodev_openssl_create(const char *name, - struct rte_crypto_vdev_init_params *init_params) + struct rte_vdev_device *vdev, + struct rte_crypto_vdev_init_params *init_params) { struct rte_cryptodev *dev; - char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct openssl_private *internals; - /* create a unique device name */ - if (create_unique_device_name(crypto_dev_name, - RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { - OPENSSL_LOG_ERR("failed to create unique cryptodev name"); - return -EINVAL; - } + if (init_params->name[0] == '\0') + snprintf(init_params->name, sizeof(init_params->name), + "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, + dev = rte_cryptodev_vdev_pmd_init(init_params->name, sizeof(struct openssl_private), - init_params->socket_id); + init_params->socket_id, + vdev); if (dev == NULL) { OPENSSL_LOG_ERR("failed to create cryptodev vdev"); goto init_error; @@ -996,7 +1294,8 @@ cryptodev_openssl_create(const char *name, dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | - RTE_CRYPTODEV_FF_CPU_AESNI; + RTE_CRYPTODEV_FF_CPU_AESNI | + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; /* Set vector instructions mode supported */ internals = dev->data->dev_private; @@ -1007,39 +1306,53 @@ cryptodev_openssl_create(const char *name, return 0; init_error: - OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", name); + OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", + init_params->name); - cryptodev_openssl_remove(crypto_dev_name); + cryptodev_openssl_remove(vdev); return -EFAULT; } /** Initialise OPENSSL crypto device */ static int -cryptodev_openssl_probe(const char *name, - const char *input_args) +cryptodev_openssl_probe(struct rte_vdev_device *vdev) { struct rte_crypto_vdev_init_params init_params = { RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, - rte_socket_id() + rte_socket_id(), + {0} }; + const char *name; + const char *input_args; - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + input_args = rte_vdev_device_args(vdev); + + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); + if (init_params.name[0] != '\0') + RTE_LOG(INFO, PMD, " User defined name = %s\n", + init_params.name); RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", init_params.max_nb_queue_pairs); RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", init_params.max_nb_sessions); - return cryptodev_openssl_create(name, &init_params); + return cryptodev_openssl_create(name, vdev, &init_params); } /** Uninitialise OPENSSL crypto device */ static int -cryptodev_openssl_remove(const char *name) +cryptodev_openssl_remove(struct rte_vdev_device *vdev) { + const char *name; + + name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL;