From 91614c73b67cad33750b04b9904dd99666b9a91e Mon Sep 17 00:00:00 2001 From: Fiona Trahe Date: Wed, 13 Jun 2018 14:14:19 +0200 Subject: [PATCH] crypto/qat: make response process function inline Optimize the dequeue function by inlining the response processing function, assuming only symmetric operations are supported. Signed-off-by: Tomasz Jozwiak Signed-off-by: Fiona Trahe --- drivers/crypto/qat/qat_qp.c | 4 +- drivers/crypto/qat/qat_qp.h | 5 -- drivers/crypto/qat/qat_sym.c | 127 ------------------------------ drivers/crypto/qat/qat_sym.h | 131 ++++++++++++++++++++++++++++++- drivers/crypto/qat/qat_sym_pmd.c | 1 - 5 files changed, 131 insertions(+), 137 deletions(-) diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c index 0fdec0da00..b190f2cee4 100644 --- a/drivers/crypto/qat/qat_qp.c +++ b/drivers/crypto/qat/qat_qp.c @@ -14,6 +14,7 @@ #include "qat_logs.h" #include "qat_device.h" #include "qat_qp.h" +#include "qat_sym.h" #include "adf_transport_access_macros.h" @@ -238,7 +239,6 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, qp->qat_dev_gen = qat_dev->qat_dev_gen; qp->build_request = qat_qp_conf->build_request; - qp->process_response = qat_qp_conf->process_response; qp->qat_dev = qat_dev; PMD_DRV_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s", @@ -612,7 +612,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops) while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && resp_counter != nb_ops) { - tmp_qp->process_response(ops, resp_msg); + qat_sym_process_response(ops, resp_msg); head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo_mask); diff --git a/drivers/crypto/qat/qat_qp.h b/drivers/crypto/qat/qat_qp.h index eb91884104..0b3d6d3aa9 100644 --- a/drivers/crypto/qat/qat_qp.h +++ b/drivers/crypto/qat/qat_qp.h @@ -21,9 +21,6 @@ typedef int (*build_request_t)(void *op, enum qat_device_gen qat_dev_gen); /**< Build a request from an op. */ -typedef int (*process_response_t)(void **ops, uint8_t *resp); -/**< Process a response descriptor and return the associated op. */ - /** * Structure with data needed for creation of queue pair. */ @@ -44,7 +41,6 @@ struct qat_qp_config { uint32_t cookie_size; int socket_id; build_request_t build_request; - process_response_t process_response; const char *service_str; }; @@ -83,7 +79,6 @@ struct qat_qp { uint32_t nb_descriptors; enum qat_device_gen qat_dev_gen; build_request_t build_request; - process_response_t process_response; struct qat_pci_device *qat_dev; /**< qat device this qp is on */ } __rte_cache_aligned; diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c index f613adce62..887d4ebcd0 100644 --- a/drivers/crypto/qat/qat_sym.c +++ b/drivers/crypto/qat/qat_sym.c @@ -12,44 +12,7 @@ #include #include "qat_logs.h" -#include "qat_sym_session.h" #include "qat_sym.h" -#include "qat_sym_pmd.h" - -#define BYTE_LENGTH 8 -/* bpi is only used for partial blocks of DES and AES - * so AES block len can be assumed as max len for iv, src and dst - */ -#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ - -/** Encrypt a single partial block - * Depends on openssl libcrypto - * Uses ECB+XOR to do CFB encryption, same result, more performant - */ -static inline int -bpi_cipher_encrypt(uint8_t *src, uint8_t *dst, - uint8_t *iv, int ivlen, int srclen, - void *bpi_ctx) -{ - EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; - int encrypted_ivlen; - uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; - uint8_t *encr = encrypted_iv; - - /* ECB method: encrypt the IV, then XOR this with plaintext */ - if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) - <= 0) - goto cipher_encrypt_err; - - for (; srclen != 0; --srclen, ++dst, ++src, ++encr) - *dst = *src ^ *encr; - - return 0; - -cipher_encrypt_err: - PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed"); - return -EINVAL; -} /** Decrypt a single partial block * Depends on openssl libcrypto @@ -136,62 +99,6 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx, return sym_op->cipher.data.length - last_block_len; } -static inline uint32_t -qat_bpicipher_postprocess(struct qat_sym_session *ctx, - struct rte_crypto_op *op) -{ - int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); - struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = block_len > 0 ? - sym_op->cipher.data.length % block_len : 0; - - if (last_block_len > 0 && - ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { - - /* Encrypt last block */ - uint8_t *last_block, *dst, *iv; - uint32_t last_block_offset; - - last_block_offset = sym_op->cipher.data.offset + - sym_op->cipher.data.length - last_block_len; - last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, - uint8_t *, last_block_offset); - - if (unlikely(sym_op->m_dst != NULL)) - /* out-of-place operation (OOP) */ - dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, - uint8_t *, last_block_offset); - else - dst = last_block; - - if (last_block_len < sym_op->cipher.data.length) - /* use previous block ciphertext as IV */ - iv = dst - block_len; - else - /* runt block, i.e. less than one full block */ - iv = rte_crypto_op_ctod_offset(op, uint8_t *, - ctx->cipher_iv.offset); - -#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX - rte_hexdump(stdout, "BPI: src before post-process:", last_block, - last_block_len); - if (sym_op->m_dst != NULL) - rte_hexdump(stdout, "BPI: dst before post-process:", - dst, last_block_len); -#endif - bpi_cipher_encrypt(last_block, dst, iv, block_len, - last_block_len, ctx->bpi_ctx); -#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX - rte_hexdump(stdout, "BPI: src after post-process:", last_block, - last_block_len); - if (sym_op->m_dst != NULL) - rte_hexdump(stdout, "BPI: dst after post-process:", dst, - last_block_len); -#endif - } - return sym_op->cipher.data.length - last_block_len; -} - static inline void set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, struct icp_qat_fw_la_cipher_req_params *cipher_param, @@ -659,37 +566,3 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, #endif return 0; } - -int -qat_sym_process_response(void **op, uint8_t *resp) -{ - - struct icp_qat_fw_comn_resp *resp_msg = - (struct icp_qat_fw_comn_resp *)resp; - struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t) - (resp_msg->opaque_data); - -#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX - rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, - sizeof(struct icp_qat_fw_comn_resp)); -#endif - - if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != - ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( - resp_msg->comn_hdr.comn_status)) { - - rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - } else { - struct qat_sym_session *sess = (struct qat_sym_session *) - get_session_private_data( - rx_op->sym->session, - cryptodev_qat_driver_id); - - if (sess->bpi_ctx) - qat_bpicipher_postprocess(sess, rx_op); - rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; - } - *op = (void *)rx_op; - - return 0; -} diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h index dffd5f369a..ddd0f473fc 100644 --- a/drivers/crypto/qat/qat_sym.h +++ b/drivers/crypto/qat/qat_sym.h @@ -6,8 +6,19 @@ #define _QAT_SYM_H_ #include +#include + +#include #include "qat_common.h" +#include "qat_sym_session.h" +#include "qat_sym_pmd.h" + +#define BYTE_LENGTH 8 +/* bpi is only used for partial blocks of DES and AES + * so AES block len can be assumed as max len for iv, src and dst + */ +#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ struct qat_sym_session; @@ -21,7 +32,123 @@ struct qat_sym_op_cookie { int qat_sym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie, enum qat_device_gen qat_dev_gen); -int -qat_sym_process_response(void **op, uint8_t *resp); + +/** Encrypt a single partial block + * Depends on openssl libcrypto + * Uses ECB+XOR to do CFB encryption, same result, more performant + */ +static inline int +bpi_cipher_encrypt(uint8_t *src, uint8_t *dst, + uint8_t *iv, int ivlen, int srclen, + void *bpi_ctx) +{ + EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; + int encrypted_ivlen; + uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; + uint8_t *encr = encrypted_iv; + + /* ECB method: encrypt the IV, then XOR this with plaintext */ + if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) + <= 0) + goto cipher_encrypt_err; + + for (; srclen != 0; --srclen, ++dst, ++src, ++encr) + *dst = *src ^ *encr; + + return 0; + +cipher_encrypt_err: + PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed"); + return -EINVAL; +} + +static inline uint32_t +qat_bpicipher_postprocess(struct qat_sym_session *ctx, + struct rte_crypto_op *op) +{ + int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); + struct rte_crypto_sym_op *sym_op = op->sym; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; + + if (last_block_len > 0 && + ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { + + /* Encrypt last block */ + uint8_t *last_block, *dst, *iv; + uint32_t last_block_offset; + + last_block_offset = sym_op->cipher.data.offset + + sym_op->cipher.data.length - last_block_len; + last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, + uint8_t *, last_block_offset); + + if (unlikely(sym_op->m_dst != NULL)) + /* out-of-place operation (OOP) */ + dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, + uint8_t *, last_block_offset); + else + dst = last_block; + + if (last_block_len < sym_op->cipher.data.length) + /* use previous block ciphertext as IV */ + iv = dst - block_len; + else + /* runt block, i.e. less than one full block */ + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "BPI: src before post-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst before post-process:", + dst, last_block_len); +#endif + bpi_cipher_encrypt(last_block, dst, iv, block_len, + last_block_len, ctx->bpi_ctx); +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "BPI: src after post-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst after post-process:", dst, + last_block_len); +#endif + } + return sym_op->cipher.data.length - last_block_len; +} + +static inline void +qat_sym_process_response(void **op, uint8_t *resp) +{ + + struct icp_qat_fw_comn_resp *resp_msg = + (struct icp_qat_fw_comn_resp *)resp; + struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t) + (resp_msg->opaque_data); + +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, + sizeof(struct icp_qat_fw_comn_resp)); +#endif + + if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != + ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( + resp_msg->comn_hdr.comn_status)) { + + rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + } else { + struct qat_sym_session *sess = (struct qat_sym_session *) + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + + + if (sess->bpi_ctx) + qat_bpicipher_postprocess(sess, rx_op); + rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } + *op = (void *)rx_op; +} #endif /* _QAT_SYM_H_ */ diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c index 28e579b77a..6b39b32f82 100644 --- a/drivers/crypto/qat/qat_sym_pmd.c +++ b/drivers/crypto/qat/qat_sym_pmd.c @@ -160,7 +160,6 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, qat_qp_conf.hw = qp_hw_data; qat_qp_conf.build_request = qat_sym_build_request; - qat_qp_conf.process_response = qat_sym_process_response; qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie); qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors; qat_qp_conf.socket_id = socket_id; -- 2.20.1