X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fqat%2Fqat_crypto.c;h=9d8dc77aba14532cfb33d4477aaa72bdb28f873f;hb=8aa8ee9d2d87ba1a0560cfdbe936c3fd7c5dc12b;hp=098109e497ef0004dc4062e6923d5ab1c0d739a5;hpb=9333cfba3bee4e1470344422cfb6531b8e09fceb;p=dpdk.git diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index 098109e497..9d8dc77aba 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2015-2017 Intel Corporation */ #include @@ -44,15 +15,12 @@ #include #include #include -#include #include -#include #include #include #include #include #include -#include #include #include #include @@ -60,7 +28,10 @@ #include #include #include -#include +#include +#include +#include + #include #include "qat_logs.h" @@ -69,6 +40,10 @@ #include "adf_transport_access_macros.h" #define BYTE_LENGTH 8 +/* bpi is only used for partial blocks of DES and AES + * so AES block len can be assumed as max len for iv, src and dst + */ +#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ static int qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo, @@ -121,16 +96,16 @@ bpi_cipher_encrypt(uint8_t *src, uint8_t *dst, { EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; int encrypted_ivlen; - uint8_t encrypted_iv[16]; - int i; + uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; + uint8_t *encr = encrypted_iv; /* ECB method: encrypt the IV, then XOR this with plaintext */ if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) <= 0) goto cipher_encrypt_err; - for (i = 0; i < srclen; i++) - *(dst+i) = *(src+i)^(encrypted_iv[i]); + for (; srclen != 0; --srclen, ++dst, ++src, ++encr) + *dst = *src ^ *encr; return 0; @@ -150,16 +125,16 @@ bpi_cipher_decrypt(uint8_t *src, uint8_t *dst, { EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; int encrypted_ivlen; - uint8_t encrypted_iv[16]; - int i; + uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; + uint8_t *encr = encrypted_iv; /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */ if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) <= 0) goto cipher_decrypt_err; - for (i = 0; i < srclen; i++) - *(dst+i) = *(src+i)^(encrypted_iv[i]); + for (; srclen != 0; --srclen, ++dst, ++src, ++encr) + *dst = *src ^ *encr; return 0; @@ -171,16 +146,19 @@ cipher_decrypt_err: /** Creates a context in either AES or DES in ECB mode * Depends on openssl libcrypto */ -static void * +static int bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, enum rte_crypto_cipher_operation direction __rte_unused, - uint8_t *key) + uint8_t *key, void **ctx) { const EVP_CIPHER *algo = NULL; - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + int ret; + *ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) + if (*ctx == NULL) { + ret = -ENOMEM; goto ctx_init_err; + } if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI) algo = EVP_des_ecb(); @@ -188,15 +166,17 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, algo = EVP_aes_128_ecb(); /* IV will be ECB encrypted whether direction is encrypt or decrypt*/ - if (EVP_EncryptInit_ex(ctx, algo, NULL, key, 0) != 1) + if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) { + ret = -EINVAL; goto ctx_init_err; + } - return ctx; + return 0; ctx_init_err: - if (ctx != NULL) - EVP_CIPHER_CTX_free(ctx); - return NULL; + if (*ctx != NULL) + EVP_CIPHER_CTX_free(*ctx); + return ret; } /** Frees a context previously created @@ -214,7 +194,7 @@ adf_modulo(uint32_t data, uint32_t shift); static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie); + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp); void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, @@ -248,10 +228,21 @@ qat_get_cmd_id(const struct rte_crypto_sym_xform *xform) /* AEAD */ if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + /* AES-GCM and AES-CCM works with different direction + * GCM first encrypts and generate hash where AES-CCM + * first generate hash and encrypts. Similar relation + * applies to decryption. + */ if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) - return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + else + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; else - return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + else + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; } if (xform->next == NULL) @@ -295,13 +286,15 @@ qat_get_cipher_xform(struct rte_crypto_sym_xform *xform) return NULL; } -void * + +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private) + struct rte_crypto_sym_xform *xform, + struct qat_session *session) { - struct qat_session *session = session_private; struct qat_pmd_private *internals = dev->data->dev_private; struct rte_crypto_cipher_xform *cipher_xform = NULL; + int ret; /* Get cipher xform from crypto xform chain */ cipher_xform = qat_get_cipher_xform(xform); @@ -314,6 +307,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -322,6 +316,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; @@ -330,6 +325,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_snow3g_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; @@ -341,6 +337,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_kasumi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE; @@ -349,6 +346,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -357,6 +355,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -365,38 +364,43 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; break; case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create DES BPI ctx"); goto error_out; } if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; break; case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create AES BPI ctx"); goto error_out; } if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -407,11 +411,13 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_cipher_algorithm_strings [cipher_xform->algo]); + ret = -ENOTSUP; goto error_out; } if (qat_alg_validate_zuc_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; @@ -423,10 +429,12 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, case RTE_CRYPTO_CIPHER_ARC4: PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u", cipher_xform->algo); + ret = -ENOTSUP; goto error_out; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n", cipher_xform->algo); + ret = -EINVAL; goto error_out; } @@ -437,17 +445,19 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_aead_session_create_content_desc_cipher(session, cipher_xform->key.data, - cipher_xform->key.length)) + cipher_xform->key.length)) { + ret = -EINVAL; goto error_out; + } - return session; + return 0; error_out: if (session->bpi_ctx) { bpi_cipher_ctx_free(session->bpi_ctx); session->bpi_ctx = NULL; } - return NULL; + return ret; } int @@ -457,20 +467,22 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, struct rte_mempool *mempool) { void *sess_private_data; + int ret; if (rte_mempool_get(mempool, &sess_private_data)) { CDEV_LOG_ERR( "Couldn't get object from session mempool"); - return -1; + return -ENOMEM; } - if (qat_crypto_set_session_parameters(dev, xform, sess_private_data) != 0) { + ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure " "session parameters"); /* Return session to mempool */ rte_mempool_put(mempool, sess_private_data); - return -1; + return ret; } set_session_private_data(sess, dev->driver_id, @@ -484,48 +496,67 @@ qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_session *session = session_private; + int ret; int qat_cmd_id; PMD_INIT_FUNC_TRACE(); /* Set context descriptor physical address */ - session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + session->cd_paddr = rte_mempool_virt2iova(session) + offsetof(struct qat_session, cd); + session->min_qat_dev_gen = QAT_GEN1; + /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) { PMD_DRV_LOG(ERR, "Unsupported xform chain requested"); - goto error_out; + return -ENOTSUP; } session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id; switch (session->qat_cmd) { case ICP_QAT_FW_LA_CMD_CIPHER: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_AUTH: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_CIPHER_HASH: - if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) - session = qat_crypto_sym_configure_session_aead(xform, + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, session); - else { - session = qat_crypto_sym_configure_session_cipher(dev, + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); - session = qat_crypto_sym_configure_session_auth(dev, + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (ret < 0) + return ret; } break; case ICP_QAT_FW_LA_CMD_HASH_CIPHER: - if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) - session = qat_crypto_sym_configure_session_aead(xform, + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, session); - else { - session = qat_crypto_sym_configure_session_auth(dev, + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); - session = qat_crypto_sym_configure_session_cipher(dev, + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (ret < 0) + return ret; } break; case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM: @@ -539,26 +570,21 @@ qat_crypto_set_session_parameters(struct rte_cryptodev *dev, case ICP_QAT_FW_LA_CMD_DELIMITER: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; } return 0; - -error_out: - return -1; } -struct qat_session * +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private) + struct qat_session *session) { - - struct qat_session *session = session_private; struct rte_crypto_auth_xform *auth_xform = NULL; struct qat_pmd_private *internals = dev->data->dev_private; auth_xform = qat_get_auth_xform(xform); @@ -588,7 +614,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, if (qat_alg_validate_aes_key(auth_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES key size"); - goto error_out; + return -EINVAL; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; @@ -611,7 +637,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_auth_algorithm_strings [auth_xform->algo]); - goto error_out; + return -ENOTSUP; } session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3; break; @@ -625,11 +651,11 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_AES_CBC_MAC: PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u", auth_xform->algo); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified", auth_xform->algo); - goto error_out; + return -EINVAL; } session->auth_iv.offset = auth_xform->iv.offset; @@ -646,7 +672,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, if (qat_alg_aead_session_create_content_desc_cipher(session, auth_xform->key.data, auth_xform->key.length)) - goto error_out; + return -EINVAL; if (qat_alg_aead_session_create_content_desc_auth(session, key_data, @@ -654,7 +680,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; } else { session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER; session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; @@ -668,12 +694,12 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; if (qat_alg_aead_session_create_content_desc_cipher(session, auth_xform->key.data, auth_xform->key.length)) - goto error_out; + return -EINVAL; } /* Restore to authentication only only */ session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH; @@ -684,22 +710,19 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; } session->digest_length = auth_xform->digest_length; - return session; - -error_out: - return NULL; + return 0; } -struct qat_session * +int qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, - struct qat_session *session_private) + struct qat_session *session) { - struct qat_session *session = session_private; struct rte_crypto_aead_xform *aead_xform = &xform->aead; + enum rte_crypto_auth_operation crypto_operation; /* * Store AEAD IV parameters as cipher IV, @@ -713,64 +736,77 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, if (qat_alg_validate_aes_key(aead_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES key size"); - goto error_out; + return -EINVAL; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; break; case RTE_CRYPTO_AEAD_AES_CCM: - PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported AEAD alg %u", - aead_xform->algo); - goto error_out; + if (qat_alg_validate_aes_key(aead_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC; + break; default: PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", aead_xform->algo); - goto error_out; + return -EINVAL; } - if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) || + (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) { session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; /* * It needs to create cipher desc content first, * then authentication */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY; + if (qat_alg_aead_session_create_content_desc_cipher(session, aead_xform->key.data, aead_xform->key.length)) - goto error_out; + return -EINVAL; if (qat_alg_aead_session_create_content_desc_auth(session, aead_xform->key.data, aead_xform->key.length, - aead_xform->add_auth_data_length, + aead_xform->aad_length, aead_xform->digest_length, - RTE_CRYPTO_AUTH_OP_GENERATE)) - goto error_out; + crypto_operation)) + return -EINVAL; } else { session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; /* * It needs to create authentication desc content first, * then cipher */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE; + if (qat_alg_aead_session_create_content_desc_auth(session, aead_xform->key.data, aead_xform->key.length, - aead_xform->add_auth_data_length, + aead_xform->aad_length, aead_xform->digest_length, - RTE_CRYPTO_AUTH_OP_VERIFY)) - goto error_out; + crypto_operation)) + return -EINVAL; if (qat_alg_aead_session_create_content_desc_cipher(session, aead_xform->key.data, aead_xform->key.length)) - goto error_out; + return -EINVAL; } session->digest_length = aead_xform->digest_length; - return session; - -error_out: - return NULL; + return 0; } unsigned qat_crypto_sym_get_session_private_size( @@ -890,6 +926,14 @@ qat_bpicipher_postprocess(struct qat_session *ctx, return sym_op->cipher.data.length - last_block_len; } +static inline void +txq_write_tail(struct qat_qp *qp, struct qat_queue *q) { + WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, q->tail); + q->nb_pending_requests = 0; + q->csr_tail = q->tail; +} + uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -913,10 +957,10 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, tail = queue->tail; /* Find how many can actually fit on the ring */ - overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops) - - queue->max_inflights; + tmp_qp->inflights16 += nb_ops; + overflow = tmp_qp->inflights16 - queue->max_inflights; if (overflow > 0) { - rte_atomic16_sub(&tmp_qp->inflights16, overflow); + tmp_qp->inflights16 -= overflow; nb_ops_possible = nb_ops - overflow; if (nb_ops_possible == 0) return 0; @@ -924,15 +968,14 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, while (nb_ops_sent != nb_ops_possible) { ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail, - tmp_qp->op_cookies[tail / queue->msg_size]); + tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp); if (ret != 0) { tmp_qp->stats.enqueue_err_count++; /* * This message cannot be enqueued, * decrease number of ops that wasn't sent */ - rte_atomic16_sub(&tmp_qp->inflights16, - nb_ops_possible - nb_ops_sent); + tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent; if (nb_ops_sent == 0) return 0; goto kick_tail; @@ -943,26 +986,59 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, cur_op++; } kick_tail: - WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number, - queue->hw_queue_number, tail); queue->tail = tail; tmp_qp->stats.enqueued_count += nb_ops_sent; + queue->nb_pending_requests += nb_ops_sent; + if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH || + queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) { + txq_write_tail(tmp_qp, queue); + } return nb_ops_sent; } +static inline +void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q) +{ + uint32_t old_head, new_head; + uint32_t max_head; + + old_head = q->csr_head; + new_head = q->head; + max_head = qp->nb_descriptors * q->msg_size; + + /* write out free descriptors */ + void *cur_desc = (uint8_t *)q->base_addr + old_head; + + if (new_head < old_head) { + memset(cur_desc, ADF_RING_EMPTY_SIG, max_head - old_head); + memset(q->base_addr, ADF_RING_EMPTY_SIG, new_head); + } else { + memset(cur_desc, ADF_RING_EMPTY_SIG, new_head - old_head); + } + q->nb_processed_responses = 0; + q->csr_head = new_head; + + /* write current head to CSR */ + WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, new_head); +} + uint16_t qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) { - struct qat_queue *queue; + struct qat_queue *rx_queue, *tx_queue; struct qat_qp *tmp_qp = (struct qat_qp *)qp; uint32_t msg_counter = 0; struct rte_crypto_op *rx_op; struct icp_qat_fw_comn_resp *resp_msg; + uint32_t head; - queue = &(tmp_qp->rx_q); + rx_queue = &(tmp_qp->rx_q); + tx_queue = &(tmp_qp->tx_q); + head = rx_queue->head; resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + queue->head); + ((uint8_t *)rx_queue->base_addr + head); while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && msg_counter != nb_ops) { @@ -972,7 +1048,6 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, sizeof(struct icp_qat_fw_comn_resp)); - #endif if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( @@ -989,23 +1064,26 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; } - *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG; - queue->head = adf_modulo(queue->head + - queue->msg_size, - ADF_RING_SIZE_MODULO(queue->queue_size)); + head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo); resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + - queue->head); + ((uint8_t *)rx_queue->base_addr + head); *ops = rx_op; ops++; msg_counter++; } if (msg_counter > 0) { - WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr, - queue->hw_bundle_number, - queue->hw_queue_number, queue->head); - rte_atomic16_sub(&tmp_qp->inflights16, msg_counter); + rx_queue->head = head; tmp_qp->stats.dequeued_count += msg_counter; + rx_queue->nb_processed_responses += msg_counter; + tmp_qp->inflights16 -= msg_counter; + + if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) + rxq_free_desc(tmp_qp, rx_queue); + } + /* also check if tail needs to be advanced */ + if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH && + tx_queue->tail != tx_queue->csr_tail) { + txq_write_tail(tmp_qp, tx_queue); } return msg_counter; } @@ -1016,7 +1094,7 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, { int nr = 1; - uint32_t buf_len = rte_pktmbuf_mtophys(buf) - + uint32_t buf_len = rte_pktmbuf_iova(buf) - buff_start + rte_pktmbuf_data_len(buf); list->bufers[0].addr = buff_start; @@ -1040,7 +1118,7 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, list->bufers[nr].len = rte_pktmbuf_data_len(buf); list->bufers[nr].resrvd = 0; - list->bufers[nr].addr = rte_pktmbuf_mtophys(buf); + list->bufers[nr].addr = rte_pktmbuf_iova(buf); buf_len += list->bufers[nr].len; buf = buf->next; @@ -1079,9 +1157,32 @@ set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, } } +/** Set IV for CCM is special case, 0th byte is set to q-1 + * where q is padding of nonce in 16 byte block + */ +static inline void +set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset, + struct icp_qat_fw_la_cipher_req_params *cipher_param, + struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz) +{ + rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) + + ICP_QAT_HW_CCM_NONCE_OFFSET, + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); + *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = + q - ICP_QAT_HW_CCM_NONCE_OFFSET; + + if (aad_len_field_sz) + rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET], + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); +} + static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie) + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp) { int ret = 0; struct qat_session *ctx; @@ -1117,6 +1218,14 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } + if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) { + PMD_DRV_LOG(ERR, "Session alg not supported on this device gen"); + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -EINVAL; + } + + + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -1125,9 +1234,13 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { - /* AES-GCM */ + /* AES-GCM or AES-CCM */ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || - ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || + (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 + && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE + && ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { do_aead = 1; } else { do_auth = 1; @@ -1234,6 +1347,11 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, } if (do_aead) { + /* + * This address may used for setting AAD physical pointer + * into IV offset from op + */ + rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr; if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ctx->qat_hash_alg == @@ -1247,6 +1365,87 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); } + set_cipher_iv(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, qat_req); + + } else if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) { + + /* In case of AES-CCM this may point to user selected memory + * or iv offset in cypto_op + */ + uint8_t *aad_data = op->sym->aead.aad.data; + /* This is true AAD length, it not includes 18 bytes of + * preceding data + */ + uint8_t aad_ccm_real_len = 0; + + uint8_t aad_len_field_sz = 0; + uint32_t msg_len_be = + rte_bswap32(op->sym->aead.data.length); + + if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) { + aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO; + aad_ccm_real_len = ctx->aad_len - + ICP_QAT_HW_CCM_AAD_B0_LEN - + ICP_QAT_HW_CCM_AAD_LEN_INFO; + } else { + /* + * aad_len not greater than 18, so no actual aad data, + * then use IV after op for B0 block + */ + aad_data = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + aad_phys_addr_aead = + rte_crypto_op_ctophys_offset(op, + ctx->cipher_iv.offset); + } + + uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length; + + aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(aad_len_field_sz, + ctx->digest_length, q); + + if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET + + (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE), + (uint8_t *)&msg_len_be, + ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE); + } else { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET, + (uint8_t *)&msg_len_be + + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE + - q), q); + } + + if (aad_len_field_sz > 0) { + *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] + = rte_bswap16(aad_ccm_real_len); + + if ((aad_ccm_real_len + aad_len_field_sz) + % ICP_QAT_HW_CCM_AAD_B0_LEN) { + uint8_t pad_len = 0; + uint8_t pad_idx = 0; + + pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN - + ((aad_ccm_real_len + aad_len_field_sz) % + ICP_QAT_HW_CCM_AAD_B0_LEN); + pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN + + aad_ccm_real_len + aad_len_field_sz; + memset(&aad_data[pad_idx], + 0, pad_len); + } + + } + + set_cipher_iv_ccm(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, q, + aad_len_field_sz); + } cipher_len = op->sym->aead.data.length; @@ -1254,10 +1453,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_len = op->sym->aead.data.length; auth_ofs = op->sym->aead.data.offset; - auth_param->u1.aad_adr = op->sym->aead.aad.phys_addr; + auth_param->u1.aad_adr = aad_phys_addr_aead; auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; - set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, - cipher_param, op, qat_req); min_ofs = op->sym->aead.data.offset; } @@ -1277,26 +1474,26 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, * so as not to overwrite data in dest buffer */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs); + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); dst_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_dst, min_ofs); + rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs); } else { /* In-place operation * Start DMA at nearest aligned address below min_ofs */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs) + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs) & QAT_64_BTYE_ALIGN_MASK; - if (unlikely((rte_pktmbuf_mtophys(op->sym->m_src) - + if (unlikely((rte_pktmbuf_iova(op->sym->m_src) - rte_pktmbuf_headroom(op->sym->m_src)) > src_buf_start)) { /* alignment has pushed addr ahead of start of mbuf * so revert and take the performance hit */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); } dst_buf_start = src_buf_start; @@ -1304,7 +1501,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, if (do_cipher || do_aead) { cipher_param->cipher_offset = - (uint32_t)rte_pktmbuf_mtophys_offset( + (uint32_t)rte_pktmbuf_iova_offset( op->sym->m_src, cipher_ofs) - src_buf_start; cipher_param->cipher_length = cipher_len; } else { @@ -1313,7 +1510,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, } if (do_auth || do_aead) { - auth_param->auth_off = (uint32_t)rte_pktmbuf_mtophys_offset( + auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset( op->sym->m_src, auth_ofs) - src_buf_start; auth_param->auth_len = auth_len; } else {